Update ripper documentation

README.md

@@ -97,7 +97,7 @@ See the [CONTRIBUTING.md](CONTRIBUTING.md) file for more information. We additio
 * [Parser translation](docs/parser_translation.md)
 * [Parsing rules](docs/parsing_rules.md)
 * [Releasing](docs/releasing.md)
-* [Ripper](docs/ripper.md)
+* [Ripper translation](docs/ripper_translation.md)
 * [Ruby API](docs/ruby_api.md)
 * [RubyParser translation](docs/ruby_parser_translation.md)
 * [Serialization](docs/serialization.md)

bin/prism

@@ -47,20 +47,41 @@ module Prism
     def benchmark(argv)
       require "benchmark/ips"
       require "parser/current"
-      require "ruby_parser"
       require "ripper"
+      require "ruby_parser"
 
       filepath = argv.fetch(0) { File.expand_path("../lib/prism/node.rb", __dir__) }
       source = File.read(filepath)
 
       Benchmark.ips do |x|
-        x.report("Prism") { Prism.parse(source, filepath: filepath) }
-        x.report("Ripper::SexpBuilder") { Ripper.sexp_raw(source, filepath) }
-        x.report("Prism::Translation::Ripper::SexpBuilder") { Prism::Translation::Ripper.sexp_raw(source, filepath) }
-        x.report("Parser::CurrentRuby") { Parser::CurrentRuby.parse(source, filepath) }
-        x.report("Prism::Translation::Parser") { Prism::Translation::Parser.parse(source, filepath) }
-        x.report("RubyParser") { RubyParser.new.parse(source, filepath) }
-        x.report("Prism::Translation::RubyParser") { Prism::Translation::RubyParser.new.parse(source, filepath) }
+        x.report("Prism") do
+          Prism.parse(source, filepath: filepath)
+        end
+
+        x.report("Ripper::SexpBuilder") do
+          Ripper.sexp_raw(source, filepath)
+        end
+
+        x.report("Prism::Translation::Ripper::SexpBuilder") do
+          Prism::Translation::Ripper.sexp_raw(source, filepath)
+        end
+
+        x.report("Parser::CurrentRuby") do
+          Parser::CurrentRuby.parse(source, filepath)
+        end
+
+        x.report("Prism::Translation::Parser") do
+          Prism::Translation::Parser.parse(source, filepath)
+        end
+
+        x.report("RubyParser") do
+          RubyParser.new.parse(source, filepath)
+        end
+
+        x.report("Prism::Translation::RubyParser") do
+          Prism::Translation::RubyParser.new.parse(source, filepath)
+        end
+
         x.compare!
       end
     end

docs/ripper_translation.md

@@ -0,0 +1,50 @@
+# Ripper translation
+
+Prism provides the ability to mirror the `Ripper` standard library. You can do this by:
+
+```ruby
+require "prism/translation/ripper/shim"
+```
+
+This provides the APIs like:
+
+```ruby
+Ripper.lex
+Ripper.parse
+Ripper.sexp_raw
+Ripper.sexp
+
+Ripper::SexpBuilder
+Ripper::SexpBuilderPP
+```
+
+Briefly, `Ripper` is a streaming parser that allows you to construct your own syntax tree. As an example:
+
+```ruby
+class ArithmeticRipper < Prism::Translation::Ripper
+  def on_binary(left, operator, right)
+    left.public_send(operator, right)
+  end
+
+  def on_int(value)
+    value.to_i
+  end
+
+  def on_program(stmts)
+    stmts
+  end
+
+  def on_stmts_new
+    []
+  end
+
+  def on_stmts_add(stmts, stmt)
+    stmts << stmt
+    stmts
+  end
+end
+
+ArithmeticRipper.new("1 + 2 - 3").parse # => [0]
+```
+
+The exact names of the `on_*` methods are listed in the `Ripper` source.

lib/prism/translation/ripper.rb

@@ -54,6 +54,38 @@ def Ripper.parse(src, filename = "(ripper)", lineno = 1)
         new(src, filename, lineno).parse
       end
 
+      # Tokenizes the Ruby program and returns an array of an array,
+      # which is formatted like
+      # <code>[[lineno, column], type, token, state]</code>.
+      # The +filename+ argument is mostly ignored.
+      # By default, this method does not handle syntax errors in +src+,
+      # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
+      #
+      #   require 'ripper'
+      #   require 'pp'
+      #
+      #   pp Ripper.lex("def m(a) nil end")
+      #   #=> [[[1,  0], :on_kw,     "def", FNAME    ],
+      #        [[1,  3], :on_sp,     " ",   FNAME    ],
+      #        [[1,  4], :on_ident,  "m",   ENDFN    ],
+      #        [[1,  5], :on_lparen, "(",   BEG|LABEL],
+      #        [[1,  6], :on_ident,  "a",   ARG      ],
+      #        [[1,  7], :on_rparen, ")",   ENDFN    ],
+      #        [[1,  8], :on_sp,     " ",   BEG      ],
+      #        [[1,  9], :on_kw,     "nil", END      ],
+      #        [[1, 12], :on_sp,     " ",   END      ],
+      #        [[1, 13], :on_kw,     "end", END      ]]
+      #
+      def Ripper.lex(src, filename = "-", lineno = 1, raise_errors: false)
+        result = Prism.lex_compat(src, filepath: filename, line: lineno)
+
+        if result.failure? && raise_errors
+          raise SyntaxError, result.errors.first.message
+        else
+          result.value
+        end
+      end
+
       # This contains a table of all of the parser events and their
       # corresponding arity.
       PARSER_EVENT_TABLE = {

prism.gemspec

@@ -35,7 +35,7 @@ Gem::Specification.new do |spec|
     "docs/parser_translation.md",
     "docs/parsing_rules.md",
     "docs/releasing.md",
-    "docs/ripper.md",
+    "docs/ripper_translation.md",
     "docs/ruby_api.md",
     "docs/ruby_parser_translation.md",
     "docs/serialization.md",