wip

yui-knk · May 4, 2024 · 187f708 · 187f708
1 parent d86e584
commit 187f708
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 13 deletions.
diff --git a/internal/parse.h b/internal/parse.h
@@ -54,7 +54,7 @@ rb_parser_t *rb_ruby_parser_set_context(rb_parser_t *p, const struct rb_iseq_str
 void rb_ruby_parser_set_script_lines(rb_parser_t *p);
 void rb_ruby_parser_error_tolerant(rb_parser_t *p);
 void rb_ruby_parser_keep_tokens(rb_parser_t *p);
-typedef rb_parser_string_t*(rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int);
+typedef void(rb_parser_lex_gets_func)(struct parser_params*, rb_parser_input_data, int, struct rb_parser_string_source*);
 rb_ast_t *rb_parser_compile(rb_parser_t *p, rb_parser_lex_gets_func *gets, const char *fname_ptr, long fname_len, rb_encoding *fname_enc, rb_parser_input_data input, int line);
 
 RUBY_SYMBOL_EXPORT_BEGIN

diff --git a/internal/ruby_parser.h b/internal/ruby_parser.h
@@ -25,7 +25,7 @@ VALUE rb_parser_new(void);
 VALUE rb_parser_compile_string_path(VALUE vparser, VALUE fname, VALUE src, int line);
 VALUE rb_str_new_parser_string(rb_parser_string_t *str);
 VALUE rb_str_new_mutable_parser_string(rb_parser_string_t *str);
-rb_parser_string_t *rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str);
+void rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str, struct rb_parser_string_source *source);
 
 VALUE rb_node_str_string_val(const NODE *);
 VALUE rb_node_sym_string_val(const NODE *);

diff --git a/parse.y b/parse.y
@@ -7795,8 +7795,12 @@ must_be_ascii_compatible(struct parser_params *p, rb_parser_string_t *s)
 static rb_parser_string_t *
 lex_getline(struct parser_params *p)
 {
-    rb_parser_string_t *line = (*p->lex.gets)(p, p->lex.input, p->line_count);
-    if (!line) return 0;
+    rb_parser_string_t *line;
+    rb_parser_string_source_t source = {0};
+
+    (*p->lex.gets)(p, p->lex.input, p->line_count, &source);
+    if (!source.ptr) return 0;
+    line = rb_parser_encoding_string_new(p, source.ptr, source.len, source.enc);
     p->line_count++;
     string_buffer_append(p, line);
     must_be_ascii_compatible(p, line);

diff --git a/ruby_parser.c b/ruby_parser.c
@@ -630,8 +630,8 @@ rb_parser_keep_tokens(VALUE vparser)
     rb_ruby_parser_keep_tokens(parser->parser_params);
 }
 
-rb_parser_string_t *
-rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str)
+void
+rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_str, struct rb_parser_string_source *source)
 {
     char *beg, *end, *start;
     long len;
@@ -641,20 +641,23 @@ rb_parser_lex_get_str(struct parser_params *p, struct lex_pointer_string *ptr_st
     len = RSTRING_LEN(s);
     start = beg;
     if (ptr_str->ptr) {
-        if (len == ptr_str->ptr) return 0;
+        if (len == ptr_str->ptr) return;
         beg += ptr_str->ptr;
         len -= ptr_str->ptr;
     }
     end = memchr(beg, '\n', len);
     if (end) len = ++end - beg;
     ptr_str->ptr += len;
-    return rb_str_to_parser_string(p, rb_str_subseq(s, beg - start, len));
+
+    source->ptr = RSTRING_PTR(s) + (beg - start);
+    source->len = len;
+    source->enc = rb_enc_get(s);
 }
 
-static rb_parser_string_t *
-lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count)
+static void
+lex_get_str(struct parser_params *p, rb_parser_input_data input, int line_count, struct rb_parser_string_source *source)
 {
-    return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input);
+    return rb_parser_lex_get_str(p, (struct lex_pointer_string *)input, source);
 }
 
 static void parser_aset_script_lines_for(VALUE path, rb_parser_ary_t *lines);
@@ -716,8 +719,8 @@ parser_compile_string(struct ruby_parser *parser, const char *f, VALUE s, int li
 
 VALUE rb_io_gets_internal(VALUE io);
 
-static rb_parser_string_t *
-lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count)
+static void
+lex_io_gets(struct parser_params *p, rb_parser_input_data input, int line_count, struct rb_parser_string_source *source)
 {
     VALUE io = (VALUE)input;
     VALUE line = rb_io_gets_internal(io);

diff --git a/rubyparser.h b/rubyparser.h
@@ -66,6 +66,12 @@ typedef struct rb_parser_string {
     char *ptr;
 } rb_parser_string_t;
 
+typedef struct rb_parser_string_source {
+	rb_encoding *enc;
+	long len;
+	const char *ptr;
+} rb_parser_string_source_t;
+
 enum rb_parser_shareability {
     rb_parser_shareable_none,
     rb_parser_shareable_literal,