From c637f53edd0c3f285d29d8679903d3b8bfcfbe75 Mon Sep 17 00:00:00 2001
From: Koichi ITO <koic.ito@gmail.com>
Date: Wed, 13 Mar 2024 00:00:02 +0900
Subject: [PATCH] [ruby/prism] Fix a token incompatibility for
 `Prism::Translation::Parser::Lexer`

This PR fixes a token incompatibility between Parser gem and `Prism::Translation::Parser` for `tBACK_REF2`:

## Parser gem (Expected)

Returns `tBACK_REF2` token:

```console
$ bundle exec ruby -Ilib -rparser/ruby33 -ve \
'buf = Parser::Source::Buffer.new("example.rb"); buf.source = "A::`"; p Parser::Ruby33.new.tokenize(buf)[2]'
ruby 3.3.0 (2023-12-25 revision https://github.com/ruby/prism/commit/5124f9ac75) [x86_64-darwin22]
[[:tCONSTANT, ["A", #<Parser::Source::Range example.rb 0...1>]], [:tCOLON2, ["::", #<Parser::Source::Range example.rb 1...3>]],
[:tBACK_REF2, ["`", #<Parser::Source::Range example.rb 3...4>]]]
```

## `Prism::Translation::Parser` (Actual)

Previously, the parser returned `tXSTRING_BEG` token when parsing the following:

```console
$ bundle exec ruby -Ilib -rprism -rprism/translation/parser33 -ve \
'buf = Parser::Source::Buffer.new("example.rb"); buf.source = "A::`"; p Prism::Translation::Parser33.new.tokenize(buf)[2]'
ruby 3.3.0 (2023-12-25 revision https://github.com/ruby/prism/commit/5124f9ac75) [x86_64-darwin22]
[[:tCONSTANT, ["A", #<Parser::Source::Range example.rb 0...1>]], [:tCOLON2, ["::", #<Parser::Source::Range example.rb 1...3>]],
[:tXSTRING_BEG, ["`", #<Parser::Source::Range example.rb 3...4>]]]
```

After the update, the parser now returns `tBACK_REF2` token for the same input:

```console
$ bundle exec ruby -Ilib -rprism -rprism/translation/parser33 -ve \
'buf = Parser::Source::Buffer.new("example.rb"); buf.source = "A::`"; p Prism::Translation::Parser33.new.tokenize(buf)[2]'
ruby 3.3.0 (2023-12-25 revision https://github.com/ruby/prism/commit/5124f9ac75) [x86_64-darwin22]
[[:tCONSTANT, ["A", #<Parser::Source::Range example.rb 0...1>]], [:tCOLON2, ["::", #<Parser::Source::Range example.rb 1...3>]],
[:tBACK_REF2, ["`", #<Parser::Source::Range example.rb 3...4>]]]
```

This correction enables the restoration of `constants.txt` as a test case.

https://github.com/ruby/prism/commit/7f63b28f98
---
 lib/prism/translation/parser/lexer.rb | 2 +-
 test/prism/parser_test.rb             | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb
index 590adef323cd46..0f16902d400606 100644
--- a/lib/prism/translation/parser/lexer.rb
+++ b/lib/prism/translation/parser/lexer.rb
@@ -24,7 +24,7 @@ class Lexer
           AMPERSAND_DOT: :tANDDOT,
           AMPERSAND_EQUAL: :tOP_ASGN,
           BACK_REFERENCE: :tBACK_REF,
-          BACKTICK: :tXSTRING_BEG,
+          BACKTICK: :tBACK_REF2,
           BANG: :tBANG,
           BANG_EQUAL: :tNEQ,
           BANG_TILDE: :tNMATCH,
diff --git a/test/prism/parser_test.rb b/test/prism/parser_test.rb
index 9559d0ebe8327d..480a31f6815c70 100644
--- a/test/prism/parser_test.rb
+++ b/test/prism/parser_test.rb
@@ -72,7 +72,6 @@ class ParserTest < TestCase
     # output expected by the parser gem, so we'll skip them for now.
     skip_tokens = [
       "comments.txt",
-      "constants.txt",
       "heredoc_with_comment.txt",
       "heredocs_leading_whitespace.txt",
       "heredocs_nested.txt",