Move logic of creating AbstractSyntaxTree::Node#all_tokens from node.…

…c to ast.c

- Write the logic in `ast_node_all_tokens()` in ast.c
- Delete `rb_ast_tokens()` and `rb_ast_set_tokens()` in node.c
- Although we considered redesigning the structure: `ast->node_buffer->tokens`
  into `ast->tokens`, we leave it as it is because rb_ast_t is an imemo.
  (We will address it in the future)

ast.c

@@ -774,10 +774,35 @@ ast_node_last_column(rb_execution_context_t *ec, VALUE self)
 static VALUE
 ast_node_all_tokens(rb_execution_context_t *ec, VALUE self)
 {
+    long i;
     struct ASTNodeData *data;
+    rb_parser_ary_t *parser_tokens;
+    rb_parser_ast_token_t *parser_token;
+    VALUE str, loc, token, all_tokens;
+
     TypedData_Get_Struct(self, struct ASTNodeData, &rb_node_type, data);
 
-    return rb_ast_tokens(data->ast);
+    parser_tokens = data->ast->node_buffer->tokens;
+    if (parser_tokens == NULL) {
+        return Qnil;
+    }
+
+    all_tokens = rb_ary_new2(parser_tokens->len);
+    for (i = 0; i < parser_tokens->len; i++) {
+        parser_token = (rb_parser_ast_token_t *)parser_tokens->data[i];
+        str = rb_str_new(parser_token->str->ptr, parser_token->str->len);
+        loc = rb_ary_new_from_args(4,
+            INT2FIX(parser_token->loc.beg_pos.lineno),
+            INT2FIX(parser_token->loc.beg_pos.column),
+            INT2FIX(parser_token->loc.end_pos.lineno),
+            INT2FIX(parser_token->loc.end_pos.column)
+        );
+        token = rb_ary_new_from_args(4, INT2FIX(parser_token->id), ID2SYM(rb_intern(parser_token->type_name)), str, loc);
+        rb_ary_push(all_tokens, token);
+    }
+    rb_obj_freeze(all_tokens);
+
+    return all_tokens;
 }
 
 static VALUE

node.c

@@ -94,14 +94,6 @@ rb_node_buffer_new(void)
 #define rb_hash_aset ast->node_buffer->config->hash_aset
 #define rb_hash_delete ast->node_buffer->config->hash_delete
 #define RB_OBJ_WRITE(old, slot, young) ast->node_buffer->config->obj_write((VALUE)(old), (VALUE *)(slot), (VALUE)(young))
-#define rb_str_new ast->node_buffer->config->str_new
-#define rb_ary_new2 ast->node_buffer->config->ary_new2
-#define rb_ary_new_from_args ast->node_buffer->config->ary_new_from_args
-#define rb_ary_push ast->node_buffer->config->ary_push
-#define rb_obj_freeze ast->node_buffer->config->obj_freeze
-#define rb_intern ast->node_buffer->config->intern
-#define ID2SYM ast->node_buffer->config->id2sym
-#define INT2NUM ast->node_buffer->config->int2num
 #endif
 
 typedef void node_itr_t(rb_ast_t *ast, void *ctx, NODE *node);
@@ -471,43 +463,6 @@ rb_ast_dispose(rb_ast_t *ast)
     rb_ast_free(ast);
 }
 
-VALUE
-rb_ast_tokens(rb_ast_t *ast)
-{
-    long i;
-    rb_parser_ary_t *parser_tokens;
-    rb_parser_ast_token_t *parser_token;
-    VALUE str, loc, token, all_tokens;
-
-    parser_tokens = ast->node_buffer->tokens;
-    if (parser_tokens == NULL) {
-        return Qnil;
-    }
-
-    all_tokens = rb_ary_new2(parser_tokens->len);
-    for (i = 0; i < parser_tokens->len; i++) {
-        parser_token = (rb_parser_ast_token_t *)parser_tokens->data[i];
-        str = rb_str_new(parser_token->str->ptr, parser_token->str->len);
-        loc = rb_ary_new_from_args(4,
-            INT2NUM(parser_token->loc.beg_pos.lineno),
-            INT2NUM(parser_token->loc.beg_pos.column),
-            INT2NUM(parser_token->loc.end_pos.lineno),
-            INT2NUM(parser_token->loc.end_pos.column)
-        );
-        token = rb_ary_new_from_args(4, INT2NUM(parser_token->id), ID2SYM(rb_intern(parser_token->type_name)), str, loc);
-        rb_ary_push(all_tokens, token);
-    }
-    rb_obj_freeze(all_tokens);
-
-    return all_tokens;
-}
-
-void
-rb_ast_set_tokens(rb_ast_t *ast, VALUE tokens)
-{
-    RB_OBJ_WRITE(ast, &ast->node_buffer->tokens, tokens);
-}
-
 VALUE
 rb_node_set_type(NODE *n, enum node_type t)
 {

node.h

@@ -55,7 +55,6 @@ rb_ast_t *rb_ast_new(void);
 #endif
 size_t rb_ast_memsize(const rb_ast_t*);
 void rb_ast_dispose(rb_ast_t*);
-VALUE rb_ast_tokens(rb_ast_t *ast);
 #if RUBY_DEBUG
 void rb_ast_node_type_change(NODE *n, enum node_type type);
 #endif
@@ -65,7 +64,6 @@ void rb_node_init(NODE *n, enum node_type type);
 void rb_ast_mark_and_move(rb_ast_t *ast, bool reference_updating);
 void rb_ast_update_references(rb_ast_t*);
 void rb_ast_free(rb_ast_t*);
-void rb_ast_set_tokens(rb_ast_t*, VALUE);
 NODE *rb_ast_newnode(rb_ast_t*, enum node_type type, size_t size, size_t alignment);
 void rb_ast_delete_node(rb_ast_t*, NODE *n);
 rb_ast_id_table_t *rb_ast_new_local_table(rb_ast_t*, int);