Add Ripper.tokenize to translation layer

It's public API and trivial to implement.

Author: Earlopain

lib/prism/translation/ripper.rb

@@ -78,6 +78,19 @@ def self.lex(src, filename = "-", lineno = 1, raise_errors: false)
         end
       end
 
+      # Tokenizes the Ruby program and returns an array of strings.
+      # The +filename+ and +lineno+ arguments are mostly ignored, since the
+      # return value is just the tokenized input.
+      # By default, this method does not handle syntax errors in +src+,
+      # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
+      #
+      #   p Ripper.tokenize("def m(a) nil end")
+      #      # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+      #
+      def self.tokenize(...)
+        lex(...).map(&:value)
+      end
+
       # This contains a table of all of the parser events and their
       # corresponding arity.
       PARSER_EVENT_TABLE = {

test/prism/ruby/ripper_test.rb

@@ -84,6 +84,11 @@ class RipperTest < TestCase
       define_method("#{fixture.test_name}_lexer_parse") { assert_ripper_lexer_parse(fixture.read) }
     end
 
+    def test_tokenize
+      source = "foo;1;BAZ"
+      assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source))
+    end
+
     # Check that the hardcoded values don't change without us noticing.
     def test_internals
       actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort