Skip to content

Commit

Permalink
[Bug #20270] Fix --parser=prism (#10970)
Browse files Browse the repository at this point in the history 
Co-authored-by: Takashi Kokubun <takashikkbn@gmail.com>
  • Loading branch information
@peterzhu2118 @k0kubun
peterzhu2118 and k0kubun committed Jun 11, 2024
1 parent d1869cf commit 97b1bf9
Show file tree
Hide file tree
Showing 2 changed files with 130 additions and 127 deletions.
248 changes: 125 additions & 123 deletions ruby.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,6 @@ enum feature_flag_bits {
SEP \
X(parsetree_with_comment) \
SEP \
X(prism_parsetree) \
SEP \
X(insns) \
SEP \
X(insns_without_opt) \
Expand All @@ -168,7 +166,7 @@ enum dump_flag_bits {
DUMP_BIT(parsetree_with_comment)),
dump_exit_bits = (DUMP_BIT(yydebug) | DUMP_BIT(syntax) |
DUMP_BIT(parsetree) | DUMP_BIT(parsetree_with_comment) |
DUMP_BIT(prism_parsetree) | DUMP_BIT(insns) | DUMP_BIT(insns_without_opt))
DUMP_BIT(insns) | DUMP_BIT(insns_without_opt))
};

static inline void
Expand Down Expand Up @@ -355,7 +353,7 @@ usage(const char *name, int help, int highlight, int columns)

static const struct ruby_opt_message help_msg[] = {
M("--copyright", "", "print the copyright"),
M("--dump={insns|parsetree|prism_parsetree|...}[,...]", "",
M("--dump={insns|parsetree|...}[,...]", "",
"dump debug information. see below for available dump list"),
M("--enable={jit|rubyopt|...}[,...]", ", --disable={jit|rubyopt|...}[,...]",
"enable or disable features. see below for available features"),
Expand Down Expand Up @@ -1986,12 +1984,96 @@ env_var_truthy(const char *name)

rb_pid_t rb_fork_ruby(int *status);

static rb_ast_t *
process_script(ruby_cmdline_options_t *opt)
{
rb_ast_t *ast;
VALUE parser = rb_parser_new();

if (opt->dump & DUMP_BIT(yydebug)) {
rb_parser_set_yydebug(parser, Qtrue);
}

if (opt->dump & DUMP_BIT(error_tolerant)) {
rb_parser_error_tolerant(parser);
}

if (opt->e_script) {
VALUE progname = rb_progname;
rb_parser_set_context(parser, 0, TRUE);

ruby_opt_init(opt);
ruby_set_script_name(progname);
rb_parser_set_options(parser, opt->do_print, opt->do_loop,
opt->do_line, opt->do_split);
ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
}
else {
VALUE f;
int xflag = opt->xflag;
f = open_load_file(opt->script_name, &xflag);
opt->xflag = xflag != 0;
rb_parser_set_context(parser, 0, f == rb_stdin);
ast = load_file(parser, opt->script_name, f, 1, opt);
}
if (!ast->body.root) {
rb_ast_dispose(ast);
return NULL;
}
return ast;
}

static void
prism_script(ruby_cmdline_options_t *opt, pm_string_t *input, pm_options_t *options)
{
ruby_opt_init(opt);

if (strcmp(opt->script, "-") == 0) {
rb_warn("Prism support for streaming code from stdin is not currently supported");
pm_string_constant_init(input, "", 0);
pm_options_filepath_set(options, "-e");
}
else if (opt->e_script) {
pm_string_constant_init(input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
pm_options_filepath_set(options, "-e");
}
else {
pm_string_mapped_init(input, RSTRING_PTR(opt->script_name));
pm_options_filepath_set(options, RSTRING_PTR(opt->script_name));
}
}

static VALUE
prism_dump_tree(pm_string_t *input, pm_options_t *options)
{
pm_parser_t parser;
pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);

pm_node_t *node = pm_parse(&parser);

pm_buffer_t output_buffer = { 0 };

pm_prettyprint(&output_buffer, &parser, node);

VALUE tree = rb_str_new(output_buffer.value, output_buffer.length);

pm_buffer_free(&output_buffer);
pm_node_destroy(&parser, node);
pm_parser_free(&parser);

return tree;
}

static VALUE
process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
{
rb_ast_t *ast = 0;
VALUE parser;
VALUE script_name;
rb_ast_t *ast = NULL;
pm_string_t pm_input = { 0 };
pm_options_t pm_options = { 0 };

#define dispose_result() \
(ast ? rb_ast_dispose(ast) : (pm_string_free(&pm_input), pm_options_free(&pm_options)))

const rb_iseq_t *iseq;
rb_encoding *enc, *lenc;
#if UTF8_PATH
Expand Down Expand Up @@ -2162,13 +2244,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
lenc = rb_locale_encoding();
rb_enc_associate(rb_progname, lenc);
rb_obj_freeze(rb_progname);
parser = rb_parser_new();
if (opt->dump & DUMP_BIT(yydebug)) {
rb_parser_set_yydebug(parser, Qtrue);
}
if (opt->dump & DUMP_BIT(error_tolerant)) {
rb_parser_error_tolerant(parser);
}
if (opt->ext.enc.name != 0) {
opt->ext.enc.index = opt_enc_index(opt->ext.enc.name);
}
Expand All @@ -2194,7 +2269,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
ienc = enc;
#endif
}
script_name = opt->script_name;
rb_enc_associate(opt->script_name, IF_UTF8_PATH(uenc, lenc));
#if UTF8_PATH
if (uenc != lenc) {
Expand Down Expand Up @@ -2261,46 +2335,35 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
ruby_set_argv(argc, argv);
opt->sflag = process_sflag(opt->sflag);

if (!(*rb_ruby_prism_ptr())) {
if (opt->e_script) {
VALUE progname = rb_progname;
rb_encoding *eenc;
rb_parser_set_context(parser, 0, TRUE);

if (opt->src.enc.index >= 0) {
eenc = rb_enc_from_index(opt->src.enc.index);
}
else {
eenc = lenc;
#if UTF8_PATH
if (ienc) eenc = ienc;
#endif
}
if (opt->e_script) {
rb_encoding *eenc;
if (opt->src.enc.index >= 0) {
eenc = rb_enc_from_index(opt->src.enc.index);
}
else {
eenc = lenc;
#if UTF8_PATH
if (eenc != uenc) {
opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
}
if (ienc) eenc = ienc;
#endif
rb_enc_associate(opt->e_script, eenc);
ruby_opt_init(opt);
ruby_set_script_name(progname);
rb_parser_set_options(parser, opt->do_print, opt->do_loop,
opt->do_line, opt->do_split);
ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
}
else {
VALUE f;
int xflag = opt->xflag;
f = open_load_file(script_name, &xflag);
opt->xflag = xflag != 0;
rb_parser_set_context(parser, 0, f == rb_stdin);
ast = load_file(parser, opt->script_name, f, 1, opt);
#if UTF8_PATH
if (eenc != uenc) {
opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
}
#endif
rb_enc_associate(opt->e_script, eenc);
}

if (!(*rb_ruby_prism_ptr())) {
if (!(ast = process_script(opt))) return Qfalse;
}
else {
prism_script(opt, &pm_input, &pm_options);
}
ruby_set_script_name(opt->script_name);
if (dump & DUMP_BIT(yydebug)) {
dump &= ~DUMP_BIT(yydebug);
if (!dump) return Qtrue;
if ((dump & DUMP_BIT(yydebug)) && !(dump &= ~DUMP_BIT(yydebug))) {
dispose_result();
return Qtrue;
}

if (opt->ext.enc.index >= 0) {
Expand All @@ -2320,11 +2383,6 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
rb_enc_set_default_internal(Qnil);
rb_stdio_set_default_encoding();

if (!(*rb_ruby_prism_ptr()) && !ast->body.root) {
rb_ast_dispose(ast);
return Qfalse;
}

opt->sflag = process_sflag(opt->sflag);
opt->xflag = 0;

Expand All @@ -2341,60 +2399,28 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
rb_define_global_function("chomp", rb_f_chomp, -1);
}

if (dump & (DUMP_BIT(prism_parsetree))) {
pm_string_t input;
pm_options_t options = { 0 };

if (strcmp(opt->script, "-") == 0) {
int xflag = opt->xflag;
VALUE rb_source = open_load_file(opt->script_name, &xflag);
opt->xflag = xflag != 0;

rb_warn("Prism support for streaming code from stdin is not currently supported");
pm_string_constant_init(&input, RSTRING_PTR(rb_source), RSTRING_LEN(rb_source));
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
}
else if (opt->e_script) {
pm_string_constant_init(&input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
pm_options_filepath_set(&options, "-e");
if (dump & (DUMP_BIT(parsetree)|DUMP_BIT(parsetree_with_comment))) {
VALUE tree;
if (ast) {
int comment = dump & DUMP_BIT(parsetree_with_comment);
tree = rb_parser_dump_tree(ast->body.root, comment);
}
else {
pm_string_mapped_init(&input, RSTRING_PTR(opt->script_name));
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
tree = prism_dump_tree(&pm_input, &pm_options);
}

pm_parser_t parser;
pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options);

pm_node_t *node = pm_parse(&parser);
pm_buffer_t output_buffer = { 0 };

pm_prettyprint(&output_buffer, &parser, node);
rb_io_write(rb_stdout, rb_str_new((const char *) output_buffer.value, output_buffer.length));
rb_io_flush(rb_stdout);

pm_buffer_free(&output_buffer);
pm_node_destroy(&parser, node);
pm_parser_free(&parser);

pm_string_free(&input);
pm_options_free(&options);
}

if (dump & (DUMP_BIT(parsetree)|DUMP_BIT(parsetree_with_comment))) {
rb_io_write(rb_stdout, rb_parser_dump_tree(ast->body.root, dump & DUMP_BIT(parsetree_with_comment)));
rb_io_write(rb_stdout, tree);
rb_io_flush(rb_stdout);
dump &= ~DUMP_BIT(parsetree)&~DUMP_BIT(parsetree_with_comment);
if (!dump) {
rb_ast_dispose(ast);
dispose_result();
return Qtrue;
}
}

{
VALUE path = Qnil;
if (!opt->e_script && strcmp(opt->script, "-")) {
path = rb_realpath_internal(Qnil, script_name, 1);
path = rb_realpath_internal(Qnil, opt->script_name, 1);
#if UTF8_PATH
if (uenc != lenc) {
path = str_conv_enc(path, uenc, lenc);
Expand All @@ -2405,41 +2431,17 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
}
}

bool optimize = !(dump & DUMP_BIT(insns_without_opt));

if ((*rb_ruby_prism_ptr())) {
pm_string_t input;
pm_options_t options = { 0 };

if (strcmp(opt->script, "-") == 0) {
int xflag = opt->xflag;
VALUE rb_source = open_load_file(opt->script_name, &xflag);
opt->xflag = xflag != 0;

rb_warn("Prism support for streaming code from stdin is not currently supported");
pm_string_constant_init(&input, RSTRING_PTR(rb_source), RSTRING_LEN(rb_source));
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
}
else if (opt->e_script) {
pm_string_constant_init(&input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
pm_options_filepath_set(&options, "-e");
}
else {
pm_string_mapped_init(&input, RSTRING_PTR(opt->script_name));
pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
}

iseq = rb_iseq_new_main_prism(&input, &options, path);
ruby_opt_init(opt);

pm_string_free(&input);
pm_options_free(&options);
if (!ast) {
iseq = rb_iseq_new_main_prism(&pm_input, &pm_options, path);
}
else {
rb_binding_t *toplevel_binding;
GetBindingPtr(rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING")),
toplevel_binding);
const struct rb_block *base_block = toplevel_context(toplevel_binding);
iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), !(dump & DUMP_BIT(insns_without_opt)));
iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), optimize);
rb_ast_dispose(ast);
}
}
Expand Down
9 changes: 5 additions & 4 deletions test/ruby/test_rubyoptions.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,17 +287,17 @@ def test_rjit_version
end
end

def test_parser_flag
warning = /compiler based on the Prism parser is currently experimental/
PRISM_WARNING = /compiler based on the Prism parser is currently experimental/

assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), warning)
def test_parser_flag
assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), PRISM_WARNING)

assert_in_out_err(%w(--parser=parse.y -e) + ["puts :hi"], "", %w(hi), [])
assert_norun_with_rflag('--parser=parse.y', '--version', "")

assert_in_out_err(%w(--parser=notreal -e) + ["puts :hi"], "", [], /unknown parser notreal/)

assert_in_out_err(%w(--parser=prism --version), "", /\+PRISM/, warning)
assert_in_out_err(%w(--parser=prism --version), "", /\+PRISM/, PRISM_WARNING)
end

def test_eval
Expand Down Expand Up @@ -1140,6 +1140,7 @@ def test_dump_parsetree_with_rflag
assert_norun_with_rflag('--dump=parsetree', '-e', '#frozen-string-literal: true')
assert_norun_with_rflag('--dump=parsetree+error_tolerant')
assert_norun_with_rflag('--dump=parse+error_tolerant')
assert_in_out_err(%w(--parser=prism --dump=parsetree -e ""), "", /ProgramNode/, PRISM_WARNING, encoding: "UTF-8")
end

def test_dump_insns_with_rflag
Expand Down

0 comments on commit 97b1bf9

Please sign in to comment.