From 34ba70c4f927d29df5426e1644fe89f917f81420 Mon Sep 17 00:00:00 2001 From: Kevin Newton Date: Wed, 6 Mar 2024 12:07:33 -0500 Subject: [PATCH] Update ripper documentation --- README.md | 2 +- bin/prism | 37 ++++++++++++++++++------ docs/ripper.md | 36 ------------------------ docs/ripper_translation.md | 50 +++++++++++++++++++++++++++++++++ lib/prism/translation/ripper.rb | 32 +++++++++++++++++++++ prism.gemspec | 2 +- 6 files changed, 113 insertions(+), 46 deletions(-) delete mode 100644 docs/ripper.md create mode 100644 docs/ripper_translation.md diff --git a/README.md b/README.md index e3b63714b..f890f680b 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. We additio * [Parser translation](docs/parser_translation.md) * [Parsing rules](docs/parsing_rules.md) * [Releasing](docs/releasing.md) -* [Ripper](docs/ripper.md) +* [Ripper translation](docs/ripper_translation.md) * [Ruby API](docs/ruby_api.md) * [RubyParser translation](docs/ruby_parser_translation.md) * [Serialization](docs/serialization.md) diff --git a/bin/prism b/bin/prism index 58b69f92d..d3bbb0897 100755 --- a/bin/prism +++ b/bin/prism @@ -47,20 +47,41 @@ module Prism def benchmark(argv) require "benchmark/ips" require "parser/current" - require "ruby_parser" require "ripper" + require "ruby_parser" filepath = argv.fetch(0) { File.expand_path("../lib/prism/node.rb", __dir__) } source = File.read(filepath) Benchmark.ips do |x| - x.report("Prism") { Prism.parse(source, filepath: filepath) } - x.report("Ripper::SexpBuilder") { Ripper.sexp_raw(source, filepath) } - x.report("Prism::Translation::Ripper::SexpBuilder") { Prism::Translation::Ripper.sexp_raw(source, filepath) } - x.report("Parser::CurrentRuby") { Parser::CurrentRuby.parse(source, filepath) } - x.report("Prism::Translation::Parser") { Prism::Translation::Parser.parse(source, filepath) } - x.report("RubyParser") { RubyParser.new.parse(source, filepath) } - x.report("Prism::Translation::RubyParser") { Prism::Translation::RubyParser.new.parse(source, filepath) } + x.report("Prism") do + Prism.parse(source, filepath: filepath) + end + + x.report("Ripper::SexpBuilder") do + Ripper.sexp_raw(source, filepath) + end + + x.report("Prism::Translation::Ripper::SexpBuilder") do + Prism::Translation::Ripper.sexp_raw(source, filepath) + end + + x.report("Parser::CurrentRuby") do + Parser::CurrentRuby.parse(source, filepath) + end + + x.report("Prism::Translation::Parser") do + Prism::Translation::Parser.parse(source, filepath) + end + + x.report("RubyParser") do + RubyParser.new.parse(source, filepath) + end + + x.report("Prism::Translation::RubyParser") do + Prism::Translation::RubyParser.new.parse(source, filepath) + end + x.compare! end end diff --git a/docs/ripper.md b/docs/ripper.md deleted file mode 100644 index e39d9e344..000000000 --- a/docs/ripper.md +++ /dev/null @@ -1,36 +0,0 @@ -# Ripper - -To test the parser, we compare against the output from `Ripper`, both for testing the lexer and testing the parser. The lexer test suite is much more feature complete at the moment. - -To lex source code using `prism`, you typically would run `Prism.lex(source)`. If you want to instead get output that `Ripper` would normally produce, you can run `Prism.lex_compat(source)`. This will produce tokens that should be equivalent to `Ripper`. - -To parse source code using `prism`, you typically would run `Prism.parse(source)`. If you want to instead using the `Ripper` streaming interface, you can inherit from `Prism::RipperCompat` and override the `on_*` methods. This will produce a syntax tree that should be equivalent to `Ripper`. That would look like: - -```ruby -class ArithmeticRipper < Prism::RipperCompat - def on_binary(left, operator, right) - left.public_send(operator, right) - end - - def on_int(value) - value.to_i - end - - def on_program(stmts) - stmts - end - - def on_stmts_new - [] - end - - def on_stmts_add(stmts, stmt) - stmts << stmt - stmts - end -end - -ArithmeticRipper.new("1 + 2 - 3").parse # => [0] -``` - -There are also APIs for building trees similar to the s-expression builders in `Ripper`. The method names are the same. These include `Prism::RipperCompat.sexp_raw(source)` and `Prism::RipperCompat.sexp(source)`. diff --git a/docs/ripper_translation.md b/docs/ripper_translation.md new file mode 100644 index 000000000..5fb04437c --- /dev/null +++ b/docs/ripper_translation.md @@ -0,0 +1,50 @@ +# Ripper translation + +Prism provides the ability to mirror the `Ripper` standard library. You can do this by: + +```ruby +require "prism/translation/ripper/shim" +``` + +This provides the APIs like: + +```ruby +Ripper.lex +Ripper.parse +Ripper.sexp_raw +Ripper.sexp + +Ripper::SexpBuilder +Ripper::SexpBuilderPP +``` + +Briefly, `Ripper` is a streaming parser that allows you to construct your own syntax tree. As an example: + +```ruby +class ArithmeticRipper < Prism::Translation::Ripper + def on_binary(left, operator, right) + left.public_send(operator, right) + end + + def on_int(value) + value.to_i + end + + def on_program(stmts) + stmts + end + + def on_stmts_new + [] + end + + def on_stmts_add(stmts, stmt) + stmts << stmt + stmts + end +end + +ArithmeticRipper.new("1 + 2 - 3").parse # => [0] +``` + +The exact names of the `on_*` methods are listed in the `Ripper` source. diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index 7888f3b8d..5079b9702 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -54,6 +54,38 @@ def Ripper.parse(src, filename = "(ripper)", lineno = 1) new(src, filename, lineno).parse end + # Tokenizes the Ruby program and returns an array of an array, + # which is formatted like + # [[lineno, column], type, token, state]. + # The +filename+ argument is mostly ignored. + # By default, this method does not handle syntax errors in +src+, + # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+. + # + # require 'ripper' + # require 'pp' + # + # pp Ripper.lex("def m(a) nil end") + # #=> [[[1, 0], :on_kw, "def", FNAME ], + # [[1, 3], :on_sp, " ", FNAME ], + # [[1, 4], :on_ident, "m", ENDFN ], + # [[1, 5], :on_lparen, "(", BEG|LABEL], + # [[1, 6], :on_ident, "a", ARG ], + # [[1, 7], :on_rparen, ")", ENDFN ], + # [[1, 8], :on_sp, " ", BEG ], + # [[1, 9], :on_kw, "nil", END ], + # [[1, 12], :on_sp, " ", END ], + # [[1, 13], :on_kw, "end", END ]] + # + def Ripper.lex(src, filename = "-", lineno = 1, raise_errors: false) + result = Prism.lex_compat(src, filepath: filename, line: lineno) + + if result.failure? && raise_errors + raise SyntaxError, result.errors.first.message + else + result.value + end + end + # This contains a table of all of the parser events and their # corresponding arity. PARSER_EVENT_TABLE = { diff --git a/prism.gemspec b/prism.gemspec index c537ddba2..1ad2da35c 100644 --- a/prism.gemspec +++ b/prism.gemspec @@ -35,7 +35,7 @@ Gem::Specification.new do |spec| "docs/parser_translation.md", "docs/parsing_rules.md", "docs/releasing.md", - "docs/ripper.md", + "docs/ripper_translation.md", "docs/ruby_api.md", "docs/ruby_parser_translation.md", "docs/serialization.md",