[ruby/prism] Add necessary encoding flags for symbols and regex

This doesn't actually fix the encodings for symbols and regex,
unfortunately. But I wanted to get this change in because it is
the last AST change we're going to make before 3.3 is released.

So, if consumers want, they can start to check these flags to
determine the encoding, even though it will be wrong. Then once we
actually set them correctly, everything should work.

ruby/prism@9b35f7e891

prism/config.yml

@@ -392,6 +392,12 @@ flags:
         comment: "s - forces the Windows-31J encoding"
       - name: UTF_8
         comment: "u - forces the UTF-8 encoding"
+      - name: FORCED_UTF8_ENCODING
+        comment: "internal bytes forced the encoding to UTF-8"
+      - name: FORCED_BINARY_ENCODING
+        comment: "internal bytes forced the encoding to binary"
+      - name: FORCED_US_ASCII_ENCODING
+        comment: "internal bytes forced the encoding to US-ASCII"
     comment: Flags for regular expression and match last line nodes.
   - name: StringFlags
     values:
@@ -402,6 +408,15 @@ flags:
       - name: FROZEN
         comment: "frozen by virtue of a `frozen_string_literal` comment"
     comment: Flags for string nodes.
+  - name: SymbolFlags
+    values:
+      - name: FORCED_UTF8_ENCODING
+        comment: "internal bytes forced the encoding to UTF-8"
+      - name: FORCED_BINARY_ENCODING
+        comment: "internal bytes forced the encoding to binary"
+      - name: FORCED_US_ASCII_ENCODING
+        comment: "internal bytes forced the encoding to US-ASCII"
+    comment: Flags for symbol nodes.
 nodes:
   - name: AliasGlobalVariableNode
     fields:
@@ -2465,6 +2480,9 @@ nodes:
           ^^^^^^^^^^^^^^
   - name: SymbolNode
     fields:
+      - name: flags
+        type: flags
+        kind: SymbolFlags
       - name: opening_loc
         type: location?
       - name: value_loc

test/prism/errors_test.rb

@@ -407,7 +407,7 @@ def test_splat_argument_after_keyword_argument
         ArgumentsNode(0, [
           KeywordHashNode(
             [AssocNode(
-              SymbolNode(nil, Location(), Location(), "foo"),
+              SymbolNode(0, nil, Location(), Location(), "foo"),
               expression("bar"),
               nil
             )]
@@ -585,16 +585,16 @@ def test_cannot_assign_to_a_reserved_numbered_parameter
       expected = BeginNode(
         Location(),
         StatementsNode([
-          LocalVariableWriteNode(:_1, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_2, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_3, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_4, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_5, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_6, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_7, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_8, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_9, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location()),
-          LocalVariableWriteNode(:_10, 0, Location(), SymbolNode(Location(), Location(), nil, "a"), Location())
+          LocalVariableWriteNode(:_1, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_2, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_3, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_4, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_5, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_6, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_7, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_8, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_9, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location()),
+          LocalVariableWriteNode(:_10, 0, Location(), SymbolNode(0, Location(), Location(), nil, "a"), Location())
         ]),
         nil,
         nil,
@@ -995,7 +995,7 @@ def test_returning_to_optional_parameters_multiple_times
 
     def test_case_without_when_clauses_errors_on_else_clause
       expected = CaseMatchNode(
-        SymbolNode(Location(), Location(), nil, "a"),
+        SymbolNode(0, Location(), Location(), nil, "a"),
         [],
         ElseNode(Location(), nil, Location()),
         Location(),
@@ -1009,7 +1009,7 @@ def test_case_without_when_clauses_errors_on_else_clause
 
     def test_case_without_clauses
       expected = CaseNode(
-        SymbolNode(Location(), Location(), nil, "a"),
+        SymbolNode(0, Location(), Location(), nil, "a"),
         [],
         nil,
         Location(),

test/prism/snapshots/alias.txt

@@ -6,12 +6,14 @@
         ├── @ AliasMethodNode (location: (1,0)-(1,15))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (1,6)-(1,10))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (1,6)-(1,7) = ":"
         │   │   ├── value_loc: (1,7)-(1,10) = "foo"
         │   │   ├── closing_loc: ∅
         │   │   └── unescaped: "foo"
         │   ├── old_name:
         │   │   @ SymbolNode (location: (1,11)-(1,15))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (1,11)-(1,12) = ":"
         │   │   ├── value_loc: (1,12)-(1,15) = "bar"
         │   │   ├── closing_loc: ∅
@@ -20,12 +22,14 @@
         ├── @ AliasMethodNode (location: (3,0)-(3,21))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (3,6)-(3,13))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (3,6)-(3,9) = "%s["
         │   │   ├── value_loc: (3,9)-(3,12) = "abc"
         │   │   ├── closing_loc: (3,12)-(3,13) = "]"
         │   │   └── unescaped: "abc"
         │   ├── old_name:
         │   │   @ SymbolNode (location: (3,14)-(3,21))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (3,14)-(3,17) = "%s["
         │   │   ├── value_loc: (3,17)-(3,20) = "def"
         │   │   ├── closing_loc: (3,20)-(3,21) = "]"
@@ -34,12 +38,14 @@
         ├── @ AliasMethodNode (location: (5,0)-(5,19))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (5,6)-(5,12))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (5,6)-(5,8) = ":'"
         │   │   ├── value_loc: (5,8)-(5,11) = "abc"
         │   │   ├── closing_loc: (5,11)-(5,12) = "'"
         │   │   └── unescaped: "abc"
         │   ├── old_name:
         │   │   @ SymbolNode (location: (5,13)-(5,19))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (5,13)-(5,15) = ":'"
         │   │   ├── value_loc: (5,15)-(5,18) = "def"
         │   │   ├── closing_loc: (5,18)-(5,19) = "'"
@@ -67,6 +73,7 @@
         │   │   └── closing_loc: (7,15)-(7,16) = "\""
         │   ├── old_name:
         │   │   @ SymbolNode (location: (7,17)-(7,23))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (7,17)-(7,19) = ":'"
         │   │   ├── value_loc: (7,19)-(7,22) = "def"
         │   │   ├── closing_loc: (7,22)-(7,23) = "'"
@@ -83,12 +90,14 @@
         ├── @ AliasMethodNode (location: (11,0)-(11,13))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (11,6)-(11,9))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (11,6)-(11,9) = "foo"
         │   │   ├── closing_loc: ∅
         │   │   └── unescaped: "foo"
         │   ├── old_name:
         │   │   @ SymbolNode (location: (11,10)-(11,13))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (11,10)-(11,13) = "bar"
         │   │   ├── closing_loc: ∅
@@ -105,12 +114,14 @@
         ├── @ AliasMethodNode (location: (15,0)-(15,12))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (15,6)-(15,9))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (15,6)-(15,9) = "foo"
         │   │   ├── closing_loc: ∅
         │   │   └── unescaped: "foo"
         │   ├── old_name:
         │   │   @ SymbolNode (location: (15,10)-(15,12))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (15,10)-(15,12) = "if"
         │   │   ├── closing_loc: ∅
@@ -119,12 +130,14 @@
         ├── @ AliasMethodNode (location: (17,0)-(17,13))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (17,6)-(17,9))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (17,6)-(17,9) = "foo"
         │   │   ├── closing_loc: ∅
         │   │   └── unescaped: "foo"
         │   ├── old_name:
         │   │   @ SymbolNode (location: (17,10)-(17,13))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (17,10)-(17,13) = "<=>"
         │   │   ├── closing_loc: ∅
@@ -133,12 +146,14 @@
         ├── @ AliasMethodNode (location: (19,0)-(19,15))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (19,6)-(19,9))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (19,6)-(19,7) = ":"
         │   │   ├── value_loc: (19,7)-(19,9) = "=="
         │   │   ├── closing_loc: ∅
         │   │   └── unescaped: "=="
         │   ├── old_name:
         │   │   @ SymbolNode (location: (19,10)-(19,15))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: (19,10)-(19,11) = ":"
         │   │   ├── value_loc: (19,11)-(19,15) = "eql?"
         │   │   ├── closing_loc: ∅
@@ -147,12 +162,14 @@
         ├── @ AliasMethodNode (location: (21,0)-(21,9))
         │   ├── new_name:
         │   │   @ SymbolNode (location: (21,6)-(21,7))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (21,6)-(21,7) = "A"
         │   │   ├── closing_loc: ∅
         │   │   └── unescaped: "A"
         │   ├── old_name:
         │   │   @ SymbolNode (location: (21,8)-(21,9))
+        │   │   ├── flags: ∅
         │   │   ├── opening_loc: ∅
         │   │   ├── value_loc: (21,8)-(21,9) = "B"
         │   │   ├── closing_loc: ∅
@@ -161,12 +178,14 @@
         └── @ AliasMethodNode (location: (23,0)-(23,11))
             ├── new_name:
             │   @ SymbolNode (location: (23,6)-(23,8))
+            │   ├── flags: ∅
             │   ├── opening_loc: (23,6)-(23,7) = ":"
             │   ├── value_loc: (23,7)-(23,8) = "A"
             │   ├── closing_loc: ∅
             │   └── unescaped: "A"
             ├── old_name:
             │   @ SymbolNode (location: (23,9)-(23,11))
+            │   ├── flags: ∅
             │   ├── opening_loc: (23,9)-(23,10) = ":"
             │   ├── value_loc: (23,10)-(23,11) = "B"
             │   ├── closing_loc: ∅

test/prism/snapshots/arrays.txt

@@ -83,6 +83,7 @@
         │   │           └── @ AssocNode (location: (5,1)-(5,12))
         │   │               ├── key:
         │   │               │   @ SymbolNode (location: (5,1)-(5,3))
+        │   │               │   ├── flags: ∅
         │   │               │   ├── opening_loc: ∅
         │   │               │   ├── value_loc: (5,1)-(5,2) = "a"
         │   │               │   ├── closing_loc: (5,2)-(5,3) = ":"
@@ -92,11 +93,13 @@
         │   │               │   ├── flags: ∅
         │   │               │   ├── elements: (length: 2)
         │   │               │   │   ├── @ SymbolNode (location: (5,5)-(5,7))
+        │   │               │   │   │   ├── flags: ∅
         │   │               │   │   │   ├── opening_loc: (5,5)-(5,6) = ":"
         │   │               │   │   │   ├── value_loc: (5,6)-(5,7) = "b"
         │   │               │   │   │   ├── closing_loc: ∅
         │   │               │   │   │   └── unescaped: "b"
         │   │               │   │   └── @ SymbolNode (location: (5,9)-(5,11))
+        │   │               │   │       ├── flags: ∅
         │   │               │   │       ├── opening_loc: (5,9)-(5,10) = ":"
         │   │               │   │       ├── value_loc: (5,10)-(5,11) = "c"
         │   │               │   │       ├── closing_loc: ∅
@@ -110,23 +113,27 @@
         │   ├── flags: ∅
         │   ├── elements: (length: 5)
         │   │   ├── @ SymbolNode (location: (9,1)-(9,3))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (9,1)-(9,2) = ":"
         │   │   │   ├── value_loc: (9,2)-(9,3) = "a"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "a"
         │   │   ├── @ SymbolNode (location: (9,5)-(9,7))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (9,5)-(9,6) = ":"
         │   │   │   ├── value_loc: (9,6)-(9,7) = "b"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "b"
         │   │   ├── @ SymbolNode (location: (10,0)-(10,2))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (10,0)-(10,1) = ":"
         │   │   │   ├── value_loc: (10,1)-(10,2) = "c"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "c"
         │   │   ├── @ IntegerNode (location: (10,3)-(10,4))
         │   │   │   └── flags: decimal
         │   │   └── @ SymbolNode (location: (14,0)-(14,2))
+        │   │       ├── flags: ∅
         │   │       ├── opening_loc: (14,0)-(14,1) = ":"
         │   │       ├── value_loc: (14,1)-(14,2) = "d"
         │   │       ├── closing_loc: ∅
@@ -137,23 +144,27 @@
         │   ├── flags: ∅
         │   ├── elements: (length: 5)
         │   │   ├── @ SymbolNode (location: (18,1)-(18,3))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (18,1)-(18,2) = ":"
         │   │   │   ├── value_loc: (18,2)-(18,3) = "a"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "a"
         │   │   ├── @ SymbolNode (location: (18,5)-(18,7))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (18,5)-(18,6) = ":"
         │   │   │   ├── value_loc: (18,6)-(18,7) = "b"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "b"
         │   │   ├── @ SymbolNode (location: (19,0)-(19,2))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: (19,0)-(19,1) = ":"
         │   │   │   ├── value_loc: (19,1)-(19,2) = "c"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "c"
         │   │   ├── @ IntegerNode (location: (19,3)-(19,4))
         │   │   │   └── flags: decimal
         │   │   └── @ SymbolNode (location: (23,0)-(23,2))
+        │   │       ├── flags: ∅
         │   │       ├── opening_loc: (23,0)-(23,1) = ":"
         │   │       ├── value_loc: (23,1)-(23,2) = "d"
         │   │       ├── closing_loc: ∅
@@ -756,16 +767,19 @@
         │   ├── flags: ∅
         │   ├── elements: (length: 3)
         │   │   ├── @ SymbolNode (location: (62,3)-(62,6))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: ∅
         │   │   │   ├── value_loc: (62,3)-(62,6) = "one"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "one"
         │   │   ├── @ SymbolNode (location: (62,7)-(62,10))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: ∅
         │   │   │   ├── value_loc: (62,7)-(62,10) = "two"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "two"
         │   │   └── @ SymbolNode (location: (62,11)-(62,16))
+        │   │       ├── flags: ∅
         │   │       ├── opening_loc: ∅
         │   │       ├── value_loc: (62,11)-(62,16) = "three"
         │   │       ├── closing_loc: ∅
@@ -805,16 +819,19 @@
         │   ├── flags: ∅
         │   ├── elements: (length: 3)
         │   │   ├── @ SymbolNode (location: (69,3)-(69,6))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: ∅
         │   │   │   ├── value_loc: (69,3)-(69,6) = "one"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "one"
         │   │   ├── @ SymbolNode (location: (69,7)-(69,10))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: ∅
         │   │   │   ├── value_loc: (69,7)-(69,10) = "two"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "two"
         │   │   └── @ SymbolNode (location: (69,11)-(69,16))
+        │   │       ├── flags: ∅
         │   │       ├── opening_loc: ∅
         │   │       ├── value_loc: (69,11)-(69,16) = "three"
         │   │       ├── closing_loc: ∅
@@ -854,16 +871,19 @@
         │   ├── flags: ∅
         │   ├── elements: (length: 3)
         │   │   ├── @ SymbolNode (location: (76,3)-(76,6))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: ∅
         │   │   │   ├── value_loc: (76,3)-(76,6) = "one"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "one"
         │   │   ├── @ SymbolNode (location: (76,7)-(76,10))
+        │   │   │   ├── flags: ∅
         │   │   │   ├── opening_loc: ∅
         │   │   │   ├── value_loc: (76,7)-(76,10) = "two"
         │   │   │   ├── closing_loc: ∅
         │   │   │   └── unescaped: "two"
         │   │   └── @ SymbolNode (location: (76,11)-(76,16))
+        │   │       ├── flags: ∅
         │   │       ├── opening_loc: ∅
         │   │       ├── value_loc: (76,11)-(76,16) = "three"
         │   │       ├── closing_loc: ∅