From c201dbc0ada03b985e297d22d6dfa24b7bac12ce Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Fri, 27 Oct 2023 13:55:48 -0400 Subject: [PATCH] [ruby/prism] Prism.parse_inline_comments https://github.com/ruby/prism/commit/5b72f84480 --- lib/prism/ffi.rb | 25 +++++++++++++++++ prism/prism.c | 30 ++++++++++++++++++--- prism/prism.h | 8 ++++++ prism/templates/lib/prism/serialize.rb.erb | 31 +++++++++++++++------- prism/templates/src/serialize.c.erb | 4 +-- test/prism/parse_inline_comments_test.rb | 2 -- 6 files changed, 83 insertions(+), 17 deletions(-) diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index cc7d94fb3f0207..ae13474c317c04 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -70,6 +70,7 @@ def self.load_exported_functions_from(header, *functions) "prism.h", "pm_version", "pm_parse_serialize", + "pm_parse_serialize_inline_comments", "pm_lex_serialize", "pm_parse_lex_serialize" ) @@ -224,6 +225,30 @@ def self.parse_file(filepath) end end + # Mirror the Prism.parse_inline_comments API by using the serialization API. + def self.parse_inline_comments(code, filepath = nil) + LibRubyParser::PrismBuffer.with do |buffer| + metadata = [filepath.bytesize, filepath.b, 0].pack("LA*L") if filepath + LibRubyParser.pm_parse_serialize_inline_comments(code, code.bytesize, buffer.pointer, metadata) + + source = Source.new(code) + loader = Serialize::Loader.new(source, buffer.read) + + loader.load_header + loader.load_force_encoding + loader.load_comments + end + end + + # Mirror the Prism.parse_file_inline_comments API by using the serialization + # API. This uses native strings instead of Ruby strings because it allows us + # to use mmap when it is available. + def self.parse_file_inline_comments(filepath) + LibRubyParser::PrismString.with(filepath) do |string| + parse_inline_comments(string.read, filepath) + end + end + # Mirror the Prism.parse_lex API by using the serialization API. def self.parse_lex(code, filepath = nil) LibRubyParser::PrismBuffer.with do |buffer| diff --git a/prism/prism.c b/prism/prism.c index 2cbf664b5da1b6..9d40d28d0153ed 100644 --- a/prism/prism.c +++ b/prism/prism.c @@ -15689,14 +15689,18 @@ pm_parse(pm_parser_t *parser) { return parse_program(parser); } -PRISM_EXPORTED_FUNCTION void -pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { +static inline void +pm_serialize_header(pm_buffer_t *buffer) { pm_buffer_append_string(buffer, "PRISM", 5); pm_buffer_append_byte(buffer, PRISM_VERSION_MAJOR); pm_buffer_append_byte(buffer, PRISM_VERSION_MINOR); pm_buffer_append_byte(buffer, PRISM_VERSION_PATCH); pm_buffer_append_byte(buffer, PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS ? 1 : 0); +} +PRISM_EXPORTED_FUNCTION void +pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer) { + pm_serialize_header(buffer); pm_serialize_content(parser, node, buffer); pm_buffer_append_string(buffer, "\0", 1); } @@ -15710,7 +15714,27 @@ pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, cons if (metadata) pm_parser_metadata(&parser, metadata); pm_node_t *node = pm_parse(&parser); - pm_serialize(&parser, node, buffer); + + pm_serialize_header(buffer); + pm_serialize_content(&parser, node, buffer); + pm_buffer_append_byte(buffer, '\0'); + + pm_node_destroy(&parser, node); + pm_parser_free(&parser); +} + +// Parse and serialize the inline comments in the given source to the given +// buffer. +PRISM_EXPORTED_FUNCTION void +pm_parse_serialize_inline_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata) { + pm_parser_t parser; + pm_parser_init(&parser, source, size, NULL); + if (metadata) pm_parser_metadata(&parser, metadata); + + pm_node_t *node = pm_parse(&parser); + pm_serialize_header(buffer); + pm_serialize_encoding(&parser.encoding, buffer); + pm_serialize_comment_list(&parser, &parser.comment_list, buffer); pm_node_destroy(&parser, node); pm_parser_free(&parser); diff --git a/prism/prism.h b/prism/prism.h index 99a6a7e2eb72d1..227b233ea1d7b1 100644 --- a/prism/prism.h +++ b/prism/prism.h @@ -30,6 +30,10 @@ void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer); +void pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer); + +void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer); + void pm_parser_metadata(pm_parser_t *parser, const char *metadata); // The prism version and the serialization format. @@ -61,6 +65,10 @@ PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, // Parse the given source to the AST and serialize the AST to the given buffer. PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata); +// Parse and serialize the inline comments in the given source to the given +// buffer. +PRISM_EXPORTED_FUNCTION void pm_parse_serialize_inline_comments(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata); + // Lex the given source and serialize to the given buffer. PRISM_EXPORTED_FUNCTION void pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer); diff --git a/prism/templates/lib/prism/serialize.rb.erb b/prism/templates/lib/prism/serialize.rb.erb index 01588c6dae3d0b..7f8b61ddfe803b 100644 --- a/prism/templates/lib/prism/serialize.rb.erb +++ b/prism/templates/lib/prism/serialize.rb.erb @@ -50,12 +50,30 @@ module Prism define_load_node_lambdas unless RUBY_ENGINE == 'ruby' end + def load_header + raise "Invalid serialization" if io.read(5) != "PRISM" + raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION] + only_semantic_fields = io.read(1).unpack1("C") + unless only_semantic_fields == 0 + raise "Invalid serialization (location fields must be included but are not)" + end + end + def load_encoding Encoding.find(io.read(load_varint)) end + def load_force_encoding + @encoding = load_encoding + @input = input.force_encoding(@encoding).freeze + end + + def load_comments + load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) } + end + def load_metadata - comments = load_varint.times.map { Comment.new(Comment::TYPES.fetch(load_varint), load_location) } + comments = load_comments magic_comments = load_varint.times.map { MagicComment.new(load_location, load_location) } errors = load_varint.times.map { ParseError.new(load_embedded_string, load_location) } warnings = load_varint.times.map { ParseWarning.new(load_embedded_string, load_location) } @@ -89,15 +107,8 @@ module Prism end def load_nodes - raise "Invalid serialization" if io.read(5) != "PRISM" - raise "Invalid serialization" if io.read(3).unpack("C3") != [MAJOR_VERSION, MINOR_VERSION, PATCH_VERSION] - only_semantic_fields = io.read(1).unpack1("C") - unless only_semantic_fields == 0 - raise "Invalid serialization (location fields must be included but are not)" - end - - @encoding = load_encoding - @input = input.force_encoding(@encoding).freeze + load_header + load_force_encoding comments, magic_comments, errors, warnings = load_metadata diff --git a/prism/templates/src/serialize.c.erb b/prism/templates/src/serialize.c.erb index 69d6d4094fe18f..d46284d3b20651 100644 --- a/prism/templates/src/serialize.c.erb +++ b/prism/templates/src/serialize.c.erb @@ -136,7 +136,7 @@ pm_serialize_comment(pm_parser_t *parser, pm_comment_t *comment, pm_buffer_t *bu pm_buffer_append_varint(buffer, pm_ptrdifft_to_u32(comment->end - comment->start)); } -static void +void pm_serialize_comment_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t *buffer) { pm_buffer_append_varint(buffer, pm_sizet_to_u32(pm_list_size(list))); @@ -189,7 +189,7 @@ pm_serialize_diagnostic_list(pm_parser_t *parser, pm_list_t *list, pm_buffer_t * } } -static void +void pm_serialize_encoding(pm_encoding_t *encoding, pm_buffer_t *buffer) { size_t encoding_length = strlen(encoding->name); pm_buffer_append_varint(buffer, pm_sizet_to_u32(encoding_length)); diff --git a/test/prism/parse_inline_comments_test.rb b/test/prism/parse_inline_comments_test.rb index 0087e1e9af07fd..d90d0abf883727 100644 --- a/test/prism/parse_inline_comments_test.rb +++ b/test/prism/parse_inline_comments_test.rb @@ -2,8 +2,6 @@ require_relative "test_helper" -return if Prism::BACKEND == :FFI - module Prism class ParseInlineCommentsTest < TestCase def test_parse_inline_comments