[Universal parser] DeVALUE of p->debug_lines and ast->body.script_lines

ast.c

@@ -97,7 +97,7 @@ rb_ast_parse_str(VALUE str, VALUE keep_script_lines, VALUE error_tolerant, VALUE
 
     StringValue(str);
     VALUE vparser = ast_parse_new();
-    if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser, Qtrue);
+    if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser);
     if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser);
     if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser);
     ast = rb_parser_compile_string_path(vparser, Qnil, str, 1);
@@ -120,7 +120,7 @@ rb_ast_parse_file(VALUE path, VALUE keep_script_lines, VALUE error_tolerant, VAL
     f = rb_file_open_str(path, "r");
     rb_funcall(f, rb_intern("set_encoding"), 2, rb_enc_from_encoding(enc), rb_str_new_cstr("-"));
     VALUE vparser = ast_parse_new();
-    if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser, Qtrue);
+    if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser);
     if (RTEST(error_tolerant))  rb_parser_error_tolerant(vparser);
     if (RTEST(keep_tokens))  rb_parser_keep_tokens(vparser);
     ast = rb_parser_compile_file_path(vparser, Qnil, f, 1);
@@ -148,7 +148,7 @@ rb_ast_parse_array(VALUE array, VALUE keep_script_lines, VALUE error_tolerant, V
 
     array = rb_check_array_type(array);
     VALUE vparser = ast_parse_new();
-    if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser, Qtrue);
+    if (RTEST(keep_script_lines)) rb_parser_set_script_lines(vparser);
     if (RTEST(error_tolerant)) rb_parser_error_tolerant(vparser);
     if (RTEST(keep_tokens)) rb_parser_keep_tokens(vparser);
     ast = rb_parser_compile_generic(vparser, lex_array, Qnil, array, 1);
@@ -806,9 +806,9 @@ ast_node_script_lines(rb_execution_context_t *ec, VALUE self)
 {
     struct ASTNodeData *data;
     TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data);
-    VALUE ret = data->ast->body.script_lines;
-    if (!RB_TYPE_P(ret, T_ARRAY)) return Qnil;
-    return ret;
+    rb_parser_ary_t *ret = data->ast->body.script_lines;
+    if (!ret || FIXNUM_P((VALUE)ret)) return Qnil;
+    return rb_parser_build_script_lines_from(ret);
 }
 
 #include "ast.rbinc"

compile.c

@@ -1483,15 +1483,16 @@ new_child_iseq(rb_iseq_t *iseq, const NODE *const node,
     ast.root = node;
     ast.frozen_string_literal = -1;
     ast.coverage_enabled = -1;
-    ast.script_lines = ISEQ_BODY(iseq)->variable.script_lines;
+    ast.script_lines = NULL;
 
     debugs("[new_child_iseq]> ---------------------------------------\n");
     int isolated_depth = ISEQ_COMPILE_DATA(iseq)->isolated_depth;
     ret_iseq = rb_iseq_new_with_opt(&ast, name,
                                     rb_iseq_path(iseq), rb_iseq_realpath(iseq),
                                     line_no, parent,
                                     isolated_depth ? isolated_depth + 1 : 0,
-                                    type, ISEQ_COMPILE_DATA(iseq)->option);
+                                    type, ISEQ_COMPILE_DATA(iseq)->option,
+                                    ISEQ_BODY(iseq)->variable.script_lines);
     debugs("[new_child_iseq]< ---------------------------------------\n");
     return ret_iseq;
 }
@@ -8735,14 +8736,15 @@ compile_builtin_mandatory_only_method(rb_iseq_t *iseq, const NODE *node, const N
         .root = RNODE(&scope_node),
         .frozen_string_literal = -1,
         .coverage_enabled = -1,
-        .script_lines = ISEQ_BODY(iseq)->variable.script_lines,
+        .script_lines = NULL
     };
 
     ISEQ_BODY(iseq)->mandatory_only_iseq =
       rb_iseq_new_with_opt(&ast, rb_iseq_base_label(iseq),
                            rb_iseq_path(iseq), rb_iseq_realpath(iseq),
                            nd_line(line_node), NULL, 0,
-                           ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option);
+                           ISEQ_TYPE_METHOD, ISEQ_COMPILE_DATA(iseq)->option,
+                           ISEQ_BODY(iseq)->variable.script_lines);
 
     ALLOCV_END(idtmp);
     return COMPILE_OK;

imemo.c

@@ -274,7 +274,7 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating)
 {
     switch (imemo_type(obj)) {
       case imemo_ast:
-        rb_ast_mark_and_move((rb_ast_t *)obj, reference_updating);
+        // TODO: Make AST decoupled from IMEMO
 
         break;
       case imemo_callcache: {

internal/parse.h

@@ -51,7 +51,7 @@ size_t rb_ruby_parser_memsize(const void *ptr);
 
 void rb_ruby_parser_set_options(rb_parser_t *p, int print, int loop, int chomp, int split);
 rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_struct *base, int main);
-void rb_ruby_parser_set_script_lines(rb_parser_t *p, VALUE lines_array);
+void rb_ruby_parser_set_script_lines(rb_parser_t *p);
 void rb_ruby_parser_error_tolerant(rb_parser_t *p);
 rb_ast_t* rb_ruby_parser_compile_file_path(rb_parser_t *p, VALUE fname, VALUE file, int start);
 void rb_ruby_parser_keep_tokens(rb_parser_t *p);

internal/ruby_parser.h

@@ -39,9 +39,11 @@ RUBY_SYMBOL_EXPORT_END
 VALUE rb_parser_end_seen_p(VALUE);
 VALUE rb_parser_encoding(VALUE);
 VALUE rb_parser_set_yydebug(VALUE, VALUE);
+VALUE rb_parser_build_script_lines_from(rb_parser_ary_t *script_lines);
+void rb_parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *script_lines);
 void rb_parser_set_options(VALUE, int, int, int, int);
 void *rb_parser_load_file(VALUE parser, VALUE name);
-void rb_parser_set_script_lines(VALUE vparser, VALUE lines_array);
+void rb_parser_set_script_lines(VALUE vparser);
 void rb_parser_error_tolerant(VALUE vparser);
 void rb_parser_keep_tokens(VALUE vparser);
 

iseq.c

@@ -834,20 +834,21 @@ rb_iseq_new(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE realpath,
             const rb_iseq_t *parent, enum rb_iseq_type type)
 {
     return rb_iseq_new_with_opt(ast, name, path, realpath, 0, parent,
-                                0, type, &COMPILE_OPTION_DEFAULT);
+                                0, type, &COMPILE_OPTION_DEFAULT,
+                                Qnil);
 }
 
 static int
 ast_line_count(const rb_ast_body_t *ast)
 {
-    if (ast->script_lines == Qfalse) {
+    if (ast->script_lines == NULL) {
         // this occurs when failed to parse the source code with a syntax error
         return 0;
     }
-    if (RB_TYPE_P(ast->script_lines, T_ARRAY)){
-        return (int)RARRAY_LEN(ast->script_lines);
+    if (!FIXNUM_P((VALUE)ast->script_lines)) {
+        return (int)ast->script_lines->len;
     }
-    return FIX2INT(ast->script_lines);
+    return FIX2INT((VALUE)ast->script_lines);
 }
 
 static VALUE
@@ -883,7 +884,8 @@ rb_iseq_new_top(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE realpath
     iseq_new_setup_coverage(path, ast, 0);
 
     return rb_iseq_new_with_opt(ast, name, path, realpath, 0, parent, 0,
-                                ISEQ_TYPE_TOP, &COMPILE_OPTION_DEFAULT);
+                                ISEQ_TYPE_TOP, &COMPILE_OPTION_DEFAULT,
+                                Qnil);
 }
 
 /**
@@ -905,7 +907,8 @@ rb_iseq_new_main(const rb_ast_body_t *ast, VALUE path, VALUE realpath, const rb_
 
     return rb_iseq_new_with_opt(ast, rb_fstring_lit("<main>"),
                                 path, realpath, 0,
-                                parent, 0, ISEQ_TYPE_MAIN, opt ? &COMPILE_OPTION_DEFAULT : &COMPILE_OPTION_FALSE);
+                                parent, 0, ISEQ_TYPE_MAIN, opt ? &COMPILE_OPTION_DEFAULT : &COMPILE_OPTION_FALSE,
+                                Qnil);
 }
 
 /**
@@ -933,7 +936,8 @@ rb_iseq_new_eval(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE realpat
     }
 
     return rb_iseq_new_with_opt(ast, name, path, realpath, first_lineno,
-                                parent, isolated_depth, ISEQ_TYPE_EVAL, &COMPILE_OPTION_DEFAULT);
+                                parent, isolated_depth, ISEQ_TYPE_EVAL, &COMPILE_OPTION_DEFAULT,
+                                Qnil);
 }
 
 rb_iseq_t *
@@ -961,7 +965,8 @@ iseq_translate(rb_iseq_t *iseq)
 rb_iseq_t *
 rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE realpath,
                      int first_lineno, const rb_iseq_t *parent, int isolated_depth,
-                     enum rb_iseq_type type, const rb_compile_option_t *option)
+                     enum rb_iseq_type type, const rb_compile_option_t *option,
+                     VALUE script_lines)
 {
     const NODE *node = ast ? ast->root : 0;
     /* TODO: argument check */
@@ -974,10 +979,11 @@ rb_iseq_new_with_opt(const rb_ast_body_t *ast, VALUE name, VALUE path, VALUE rea
         option = set_compile_option_from_ast(&new_opt, ast);
     }
 
-    VALUE script_lines = Qnil;
-
-    if (ast && !FIXNUM_P(ast->script_lines) && ast->script_lines) {
-        script_lines = ast->script_lines;
+    if (!NIL_P(script_lines)) {
+        // noop
+    }
+    else if (ast && !FIXNUM_P((VALUE)ast->script_lines) && ast->script_lines) {
+        script_lines = rb_parser_build_script_lines_from(ast->script_lines);
     }
     else if (parent) {
         script_lines = ISEQ_BODY(parent)->variable.script_lines;
@@ -1219,7 +1225,7 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V
         const rb_iseq_t *outer_scope = rb_iseq_new(NULL, name, name, Qnil, 0, ISEQ_TYPE_TOP);
         VALUE outer_scope_v = (VALUE)outer_scope;
         rb_parser_set_context(parser, outer_scope, FALSE);
-        rb_parser_set_script_lines(parser, RBOOL(ruby_vm_keep_script_lines));
+        if (ruby_vm_keep_script_lines) rb_parser_set_script_lines(parser);
         RB_GC_GUARD(outer_scope_v);
         ast = (*parse)(parser, file, src, ln);
     }
@@ -1230,7 +1236,8 @@ rb_iseq_compile_with_option(VALUE src, VALUE file, VALUE realpath, VALUE line, V
     }
     else {
         iseq = rb_iseq_new_with_opt(&ast->body, name, file, realpath, ln,
-                                    NULL, 0, ISEQ_TYPE_TOP, &option);
+                                    NULL, 0, ISEQ_TYPE_TOP, &option,
+                                    Qnil);
         rb_ast_dispose(ast);
     }
 
@@ -1621,7 +1628,8 @@ iseqw_s_compile_file(int argc, VALUE *argv, VALUE self)
     ret = iseqw_new(rb_iseq_new_with_opt(&ast->body, rb_fstring_lit("<main>"),
                                          file,
                                          rb_realpath_internal(Qnil, file, 1),
-                                         1, NULL, 0, ISEQ_TYPE_TOP, &option));
+                                         1, NULL, 0, ISEQ_TYPE_TOP, &option,
+                                         Qnil));
     rb_ast_dispose(ast);
 
     rb_vm_pop_frame(ec);

mini_builtin.c

@@ -39,7 +39,7 @@ builtin_iseq_load(const char *feature_name, const struct rb_builtin_function *ta
         .coverage_enabled = FALSE,
         .debug_level = 0,
     };
-    const rb_iseq_t *iseq = rb_iseq_new_with_opt(&ast->body, name_str, name_str, Qnil, 0, NULL, 0, ISEQ_TYPE_TOP, &optimization);
+    const rb_iseq_t *iseq = rb_iseq_new_with_opt(&ast->body, name_str, name_str, Qnil, 0, NULL, 0, ISEQ_TYPE_TOP, &optimization, Qnil);
     GET_VM()->builtin_function_table = NULL;
 
     rb_ast_dispose(ast);

node.c

@@ -20,12 +20,13 @@
 
 #include "internal.h"
 #include "internal/hash.h"
-#include "internal/variable.h"
 #include "ruby/ruby.h"
 #include "vm_core.h"
 
 #endif
 
+#include "internal/variable.h"
+
 #define NODE_BUF_DEFAULT_SIZE (sizeof(struct RNode) * 16)
 
 static void
@@ -344,18 +345,24 @@ iterate_node_values(rb_ast_t *ast, node_buffer_list_t *nb, node_itr_t * func, vo
     }
 }
 
-void
-rb_ast_mark_and_move(rb_ast_t *ast, bool reference_updating)
+static void
+script_lines_free(rb_ast_t *ast, rb_parser_ary_t *script_lines)
 {
-    if (ast->node_buffer) {
-        if (ast->body.script_lines) rb_gc_mark_and_move(&ast->body.script_lines);
+    for (long i = 0; i < script_lines->len; i++) {
+        parser_string_free(ast, (rb_parser_string_t *)script_lines->data[i]);
     }
+    xfree(script_lines->data);
+    xfree(script_lines);
 }
 
 void
 rb_ast_free(rb_ast_t *ast)
 {
     if (ast->node_buffer) {
+        if (ast->body.script_lines && !FIXNUM_P((VALUE)ast->body.script_lines)) {
+            script_lines_free(ast, ast->body.script_lines);
+            ast->body.script_lines = NULL;
+        }
         rb_node_buffer_free(ast, ast->node_buffer);
         ast->node_buffer = 0;
     }

node.h

@@ -56,7 +56,6 @@ void rb_ast_dispose(rb_ast_t*);
 const char *ruby_node_name(int node);
 void rb_node_init(NODE *n, enum node_type type);
 
-void rb_ast_mark_and_move(rb_ast_t *ast, bool reference_updating);
 void rb_ast_update_references(rb_ast_t*);
 void rb_ast_free(rb_ast_t*);
 NODE *rb_ast_newnode(rb_ast_t*, enum node_type type, size_t size, size_t alignment);