Introduce --parser runtime flag

Introduce runtime flag for specifying the parser, ``` ruby --parser=prism ``` also update the description: ``` $ ruby --parser=prism --version ruby 3.3.0dev (2023-12-08T04:47:14Z add-parser-runtime.. 0616384) +PRISM [x86_64-darwin23] ``` [Bug #20044]
ruby · Dec 15, 2023 · 55326a9 · 55326a9
1 parent 655c027
commit 55326a9
Show file tree

Hide file tree

Showing 10 changed files with 579 additions and 53 deletions.
diff --git a/common.mk b/common.mk
diff --git a/compile.c b/compile.c
@@ -44,7 +44,6 @@
 #include "builtin.h"
 #include "insns.inc"
 #include "insns_info.inc"
-#include "prism_compile.h"
 
 #undef RUBY_UNTYPED_DATA_WARNING
 #define RUBY_UNTYPED_DATA_WARNING 0

diff --git a/iseq.c b/iseq.c
@@ -43,7 +43,6 @@
 #include "builtin.h"
 #include "insns.inc"
 #include "insns_info.inc"
-#include "prism_compile.h"
 
 VALUE rb_cISeq;
 static VALUE iseqw_new(const rb_iseq_t *iseq);
@@ -1501,6 +1500,22 @@ iseqw_s_compile_file_prism(int argc, VALUE *argv, VALUE self)
     return iseqw_new(iseq);
 }
 
+rb_iseq_t *
+rb_iseq_new_main_prism(pm_string_t *input, pm_options_t *options, VALUE path) {
+    pm_parser_t parser;
+    pm_parser_init(&parser, pm_string_source(input), pm_string_length(input), options);
+
+    if (NIL_P(path)) path = rb_fstring_lit("<compiled>");
+    int start_line = 0;
+    pm_options_line_set(options, start_line);
+
+    rb_iseq_t *iseq = iseq_alloc();
+    iseqw_s_compile_prism_compile(&parser, Qnil, iseq, path, path, start_line);
+
+    pm_parser_free(&parser);
+    return iseq;
+}
+
 /*
  *  call-seq:
  *      InstructionSequence.compile_file(file[, options]) -> iseq

diff --git a/iseq.h b/iseq.h
@@ -13,6 +13,7 @@
 #include "internal/gc.h"
 #include "shape.h"
 #include "vm_core.h"
+#include "prism_compile.h"
 
 RUBY_EXTERN const int ruby_api_version[];
 #define ISEQ_MAJOR_VERSION ((unsigned int)ruby_api_version[0])
@@ -173,6 +174,7 @@ void rb_iseq_init_trace(rb_iseq_t *iseq);
 int rb_iseq_add_local_tracepoint_recursively(const rb_iseq_t *iseq, rb_event_flag_t turnon_events, VALUE tpval, unsigned int target_line, bool target_bmethod);
 int rb_iseq_remove_local_tracepoint_recursively(const rb_iseq_t *iseq, VALUE tpval);
 const rb_iseq_t *rb_iseq_load_iseq(VALUE fname);
+rb_iseq_t * rb_iseq_new_main_prism(pm_string_t *input, pm_options_t *options, VALUE path);
 
 #if VM_INSN_INFO_TABLE_IMPL == 2
 unsigned int *rb_iseq_insns_info_decode_positions(const struct rb_iseq_constant_body *body);

diff --git a/load.c b/load.c
@@ -737,21 +737,37 @@ load_iseq_eval(rb_execution_context_t *ec, VALUE fname)
     const rb_iseq_t *iseq = rb_iseq_load_iseq(fname);
 
     if (!iseq) {
-        rb_execution_context_t *ec = GET_EC();
-        VALUE v = rb_vm_push_frame_fname(ec, fname);
-        rb_ast_t *ast;
-        VALUE parser = rb_parser_new();
-        rb_parser_set_context(parser, NULL, FALSE);
-        ast = (rb_ast_t *)rb_parser_load_file(parser, fname);
-
-        rb_thread_t *th = rb_ec_thread_ptr(ec);
-        VALUE realpath_map = get_loaded_features_realpath_map(th->vm);
-
-        iseq = rb_iseq_new_top(&ast->body, rb_fstring_lit("<top (required)>"),
-                               fname, realpath_internal_cached(realpath_map, fname), NULL);
-        rb_ast_dispose(ast);
-        rb_vm_pop_frame(ec);
-        RB_GC_GUARD(v);
+        if (*rb_ruby_prism_ptr()) {
+            pm_string_t input;
+            pm_options_t options = { 0 };
+
+            pm_string_mapped_init(&input, RSTRING_PTR(fname));
+            pm_options_filepath_set(&options, RSTRING_PTR(fname));
+
+            pm_parser_t parser;
+            pm_parser_init(&parser, pm_string_source(&input), pm_string_length(&input), &options);
+
+            iseq = rb_iseq_new_main_prism(&input, &options, fname);
+
+            pm_string_free(&input);
+            pm_options_free(&options);
+        } else {
+            rb_execution_context_t *ec = GET_EC();
+            VALUE v = rb_vm_push_frame_fname(ec, fname);
+            rb_ast_t *ast;
+            VALUE parser = rb_parser_new();
+            rb_parser_set_context(parser, NULL, FALSE);
+            ast = (rb_ast_t *)rb_parser_load_file(parser, fname);
+
+            rb_thread_t *th = rb_ec_thread_ptr(ec);
+            VALUE realpath_map = get_loaded_features_realpath_map(th->vm);
+
+            iseq = rb_iseq_new_top(&ast->body, rb_fstring_lit("<top (required)>"),
+                                   fname, realpath_internal_cached(realpath_map, fname), NULL);
+            rb_ast_dispose(ast);
+            rb_vm_pop_frame(ec);
+            RB_GC_GUARD(v);
+        }
     }
     rb_exec_event_hook_script_compiled(ec, iseq, Qnil);
     rb_iseq_eval(iseq);

diff --git a/prism_compile.h b/prism_compile.h
@@ -26,3 +26,4 @@ typedef struct pm_scope_node {
 } pm_scope_node_t;
 
 void pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_t *previous, pm_parser_t *parser);
+bool *rb_ruby_prism_ptr(void);
diff --git a/ruby.c b/ruby.c
@@ -56,7 +56,6 @@
 #include "internal/thread.h"
 #include "internal/ruby_parser.h"
 #include "internal/variable.h"
-#include "prism_compile.h"
 #include "ruby/encoding.h"
 #include "ruby/thread.h"
 #include "ruby/util.h"
@@ -362,6 +361,8 @@ usage(const char *name, int help, int highlight, int columns)
           "enable or disable features. see below for available features"),
         M("--external-encoding=encoding",           ", --internal-encoding=encoding",
           "specify the default external or internal character encoding"),
+        M("--parser={parse.y|prism}",           ", --parser=prism",
+          "the parser used to parse Ruby code (experimental)"),
         M("--backtrace-limit=num",                  "", "limit the maximum length of backtrace"),
         M("--verbose",                              "", "turn on verbose mode and disable script from stdin"),
         M("--version",                              "", "print the version number, then exit"),
@@ -1407,6 +1408,18 @@ proc_long_options(ruby_cmdline_options_t *opt, const char *s, long argc, char **
     else if (is_option_with_arg("external-encoding", Qfalse, Qtrue)) {
         set_external_encoding_once(opt, s, 0);
     }
+    else if (is_option_with_arg("parser", Qfalse, Qtrue)) {
+        if (strcmp("prism", s) == 0) {
+            (*rb_ruby_prism_ptr()) = true;
+            rb_warn("The Prism compiler is currently experimental and compatibility with parse.y is not yet complete. Please report an issues you find on the prism issue tracker.");
+        }
+        else if (strcmp("parse.y", s) == 0) {
+            // default behavior
+        } else {
+            rb_raise(rb_eRuntimeError,
+                     "unknown parser %s", s);
+        }
+    }
 #if defined ALLOW_DEFAULT_SOURCE_ENCODING && ALLOW_DEFAULT_SOURCE_ENCODING
     else if (is_option_with_arg("source-encoding", Qfalse, Qtrue)) {
         set_source_encoding_once(opt, s, 0);
@@ -2244,39 +2257,41 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
     ruby_set_argv(argc, argv);
     opt->sflag = process_sflag(opt->sflag);
 
-    if (opt->e_script) {
-        VALUE progname = rb_progname;
-        rb_encoding *eenc;
-        rb_parser_set_context(parser, 0, TRUE);
+    if (!(*rb_ruby_prism_ptr())) {
+        if (opt->e_script) {
+            VALUE progname = rb_progname;
+            rb_encoding *eenc;
+            rb_parser_set_context(parser, 0, TRUE);
 
-        if (opt->src.enc.index >= 0) {
-            eenc = rb_enc_from_index(opt->src.enc.index);
-        }
-        else {
-            eenc = lenc;
+            if (opt->src.enc.index >= 0) {
+                eenc = rb_enc_from_index(opt->src.enc.index);
+            }
+            else {
+                eenc = lenc;
 #if UTF8_PATH
-            if (ienc) eenc = ienc;
+                if (ienc) eenc = ienc;
 #endif
-        }
+            }
 #if UTF8_PATH
-        if (eenc != uenc) {
-            opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
-        }
+            if (eenc != uenc) {
+                opt->e_script = str_conv_enc(opt->e_script, uenc, eenc);
+            }
 #endif
-        rb_enc_associate(opt->e_script, eenc);
-        ruby_opt_init(opt);
-        ruby_set_script_name(progname);
-        rb_parser_set_options(parser, opt->do_print, opt->do_loop,
-                              opt->do_line, opt->do_split);
-        ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
-    }
-    else {
-        VALUE f;
-        int xflag = opt->xflag;
-        f = open_load_file(script_name, &xflag);
-        opt->xflag = xflag != 0;
-        rb_parser_set_context(parser, 0, f == rb_stdin);
-        ast = load_file(parser, opt->script_name, f, 1, opt);
+            rb_enc_associate(opt->e_script, eenc);
+            ruby_opt_init(opt);
+            ruby_set_script_name(progname);
+            rb_parser_set_options(parser, opt->do_print, opt->do_loop,
+                                  opt->do_line, opt->do_split);
+            ast = rb_parser_compile_string(parser, opt->script, opt->e_script, 1);
+        }
+        else {
+            VALUE f;
+            int xflag = opt->xflag;
+            f = open_load_file(script_name, &xflag);
+            opt->xflag = xflag != 0;
+            rb_parser_set_context(parser, 0, f == rb_stdin);
+            ast = load_file(parser, opt->script_name, f, 1, opt);
+        }
     }
     ruby_set_script_name(opt->script_name);
     if (dump & DUMP_BIT(yydebug)) {
@@ -2301,7 +2316,7 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
         rb_enc_set_default_internal(Qnil);
     rb_stdio_set_default_encoding();
 
-    if (!ast->body.root) {
+    if (!(*rb_ruby_prism_ptr()) && !ast->body.root) {
         rb_ast_dispose(ast);
         return Qfalse;
     }
@@ -2378,12 +2393,30 @@ process_options(int argc, char **argv, ruby_cmdline_options_t *opt)
         }
 
 
-        rb_binding_t *toplevel_binding;
-        GetBindingPtr(rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING")),
-                      toplevel_binding);
-        const struct rb_block *base_block = toplevel_context(toplevel_binding);
-        iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), !(dump & DUMP_BIT(insns_without_opt)));
-        rb_ast_dispose(ast);
+        if ((*rb_ruby_prism_ptr())) {
+            pm_string_t input;
+            pm_options_t options = { 0 };
+
+            if (opt->e_script) {
+                pm_string_constant_init(&input, RSTRING_PTR(opt->e_script), RSTRING_LEN(opt->e_script));
+                pm_options_filepath_set(&options, "-e");
+            } else {
+                pm_string_mapped_init(&input, RSTRING_PTR(opt->script_name));
+                pm_options_filepath_set(&options, RSTRING_PTR(opt->script_name));
+            }
+
+            iseq = rb_iseq_new_main_prism(&input, &options, path);
+
+            pm_string_free(&input);
+            pm_options_free(&options);
+        } else {
+            rb_binding_t *toplevel_binding;
+            GetBindingPtr(rb_const_get(rb_cObject, rb_intern("TOPLEVEL_BINDING")),
+                          toplevel_binding);
+            const struct rb_block *base_block = toplevel_context(toplevel_binding);
+            iseq = rb_iseq_new_main(&ast->body, opt->script_name, path, vm_block_iseq(base_block), !(dump & DUMP_BIT(insns_without_opt)));
+            rb_ast_dispose(ast);
+        }
     }
 
     if (dump & (DUMP_BIT(insns) | DUMP_BIT(insns_without_opt))) {
@@ -2940,6 +2973,8 @@ ruby_process_options(int argc, char **argv)
     VALUE iseq;
     const char *script_name = (argc > 0 && argv[0]) ? argv[0] : ruby_engine;
 
+    (*rb_ruby_prism_ptr()) = false;
+
     if (!origarg.argv || origarg.argc <= 0) {
         origarg.argc = argc;
         origarg.argv = argv;

diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb
@@ -287,6 +287,19 @@ def test_rjit_version
     end
   end
 
+  def test_parser_flag
+    warning = /The Prism compiler is currently experimental and compatibility with parse.y is not yet complete. Please report an issues you find on the prism issue tracker./
+
+    assert_in_out_err(%w(--parser=prism -e) + ["puts :hi"], "", %w(hi), warning)
+
+    assert_in_out_err(%w(--parser=parse.y -e) + ["puts :hi"], "", %w(hi), [])
+    assert_norun_with_rflag('--parser=parse.y', '--version', "")
+
+    assert_in_out_err(%w(--parser=notreal -e) + ["puts :hi"], "", [], /unknown parser notreal/)
+
+    assert_in_out_err(%w(--parser=prism --version), "", /\+PRISM/, warning)
+  end
+
   def test_eval
     assert_in_out_err(%w(-e), "", [], /no code specified for -e \(RuntimeError\)/)
   end

diff --git a/version.c b/version.c
@@ -141,6 +141,8 @@ Init_version(void)
 
 int ruby_mn_threads_enabled;
 
+bool * rb_ruby_prism_ptr(void);
+
 static void
 define_ruby_description(const char *const jit_opt)
 {
@@ -151,15 +153,18 @@ define_ruby_description(const char *const jit_opt)
         ];
 
     const char *const threads_opt = ruby_mn_threads_enabled ? " +MN" : "";
+    const char *const parser_opt = (*rb_ruby_prism_ptr()) ? " +PRISM" : "";
 
     int n = snprintf(desc, sizeof(desc),
                      "%.*s"
                      "%s" // jit_opt
                      "%s" // threads_opts
+                     "%s" // parser_opt
                      "%s",
                      ruby_description_opt_point, ruby_description,
                      jit_opt,
                      threads_opt,
+                     parser_opt,
                      ruby_description + ruby_description_opt_point);
 
     VALUE description = rb_obj_freeze(rb_usascii_str_new_static(desc, n));

diff --git a/vm.c b/vm.c
@@ -4256,6 +4256,14 @@ rb_ruby_verbose_ptr(void)
     return &cr->verbose;
 }
 
+static bool prism;
+
+bool *
+rb_ruby_prism_ptr(void)
+{
+    return &prism;
+}
+
 VALUE *
 rb_ruby_debug_ptr(void)
 {