[ruby/syntax_suggest] Initial support for the prism parser

Prism will be the parser in Ruby 3.3. We need to support 3.0+ so we will have to "dual boot" both parsers.

Todo:

- LexAll to support Prism lex output
- Add tests that exercise both Ripper and prism codepaths on CI
- Handle ruby/prism#1972 in `ripper_errors.rb`
- Update docs to not mention Ripper explicitly
- Consider different/cleaner APIs for separating out Ripper and Prism

ruby/syntax_suggest@a7d6991cc4

lib/syntax_suggest/api.rb

@@ -5,7 +5,22 @@
 require "tmpdir"
 require "stringio"
 require "pathname"
-require "ripper"
+
+# rubocop:disable Style/IdenticalConditionalBranches
+if ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] # For testing dual ripper/prism support
+  require "ripper"
+else
+  # TODO remove require
+  # Allow both to be loaded to enable more atomic commits
+  require "ripper"
+  begin
+    require "prism"
+  rescue LoadError
+    require "ripper"
+  end
+end
+# rubocop:enable Style/IdenticalConditionalBranches
+
 require "timeout"
 
 module SyntaxSuggest
@@ -16,6 +31,14 @@ module SyntaxSuggest
   class Error < StandardError; end
   TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i
 
+  # SyntaxSuggest.use_prism_parser? [Private]
+  #
+  # Tells us if the prism parser is available for use
+  # or if we should fallback to `Ripper`
+  def self.use_prism_parser?
+    defined?(Prism)
+  end
+
   # SyntaxSuggest.handle_error [Public]
   #
   # Takes a `SyntaxError` exception, uses the
@@ -129,11 +152,20 @@ def self.valid_without?(without_lines:, code_lines:)
   # SyntaxSuggest.invalid? [Private]
   #
   # Opposite of `SyntaxSuggest.valid?`
-  def self.invalid?(source)
-    source = source.join if source.is_a?(Array)
-    source = source.to_s
+  if defined?(Prism)
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
+
+      Prism.parse(source).failure?
+    end
+  else
+    def self.invalid?(source)
+      source = source.join if source.is_a?(Array)
+      source = source.to_s
 
-    Ripper.new(source).tap(&:parse).error?
+      Ripper.new(source).tap(&:parse).error?
+    end
   end
 
   # SyntaxSuggest.valid? [Private]
@@ -191,7 +223,9 @@ def self.valid?(source)
 require_relative "code_line"
 require_relative "code_block"
 require_relative "block_expand"
-require_relative "ripper_errors"
+if !SyntaxSuggest.use_prism_parser?
+  require_relative "ripper_errors"
+end
 require_relative "priority_queue"
 require_relative "unvisited_lines"
 require_relative "around_block_scan"

lib/syntax_suggest/explain_syntax.rb

@@ -3,6 +3,16 @@
 require_relative "left_right_lex_count"
 
 module SyntaxSuggest
+  class GetParseErrors
+    def self.errors(source)
+      if SyntaxSuggest.use_prism_parser?
+        Prism.parse(source).errors.map(&:message)
+      else
+        RipperErrors.new(source).call.errors
+      end
+    end
+  end
+
   # Explains syntax errors based on their source
   #
   # example:
@@ -94,7 +104,7 @@ def why(miss)
     # on the original ripper error messages
     def errors
       if missing.empty?
-        return RipperErrors.new(@code_lines.map(&:original).join).call.errors
+        return GetParseErrors.errors(@code_lines.map(&:original).join)
       end
 
       missing.map { |miss| why(miss) }

lib/syntax_suggest/lex_all.rb

@@ -11,26 +11,40 @@ class LexAll
     include Enumerable
 
     def initialize(source:, source_lines: nil)
-      @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos)
-      lineno = @lex.last.pos.first + 1
+      @lex = self.class.lex(source, 1)
+      lineno = @lex.last[0][0] + 1
       source_lines ||= source.lines
       last_lineno = source_lines.length
 
       until lineno >= last_lineno
         lines = source_lines[lineno..]
 
         @lex.concat(
-          Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos)
+          self.class.lex(lines.join, lineno + 1)
         )
-        lineno = @lex.last.pos.first + 1
+
+        lineno = @lex.last[0].first + 1
       end
 
       last_lex = nil
       @lex.map! { |elem|
-        last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex)
+        last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex)
       }
     end
 
+    # rubocop:disable Style/IdenticalConditionalBranches
+    if SyntaxSuggest.use_prism_parser?
+      def self.lex(source, line_number)
+        # Prism.lex_compat(source, line: line_number).value.sort_by {|values| values[0] }
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    else
+      def self.lex(source, line_number)
+        Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos)
+      end
+    end
+    # rubocop:enable Style/IdenticalConditionalBranches
+
     def to_a
       @lex
     end

spec/syntax_suggest/unit/explain_syntax_spec.rb

@@ -14,7 +14,11 @@ module SyntaxSuggest
       ).call
 
       expect(explain.missing).to eq([])
-      expect(explain.errors.join).to include("unterminated string")
+      if SyntaxSuggest.use_prism_parser?
+        expect(explain.errors.join).to include("Expected a closing delimiter for the interpolated string")
+      else
+        expect(explain.errors.join).to include("unterminated string")
+      end
     end
 
     it "handles %w[]" do
@@ -191,7 +195,7 @@ def meow
       ).call
 
       expect(explain.missing).to eq([])
-      expect(explain.errors).to eq(RipperErrors.new(source).call.errors)
+      expect(explain.errors).to eq(GetParseErrors.errors(source))
     end
 
     it "handles an unexpected rescue" do