Revert "Revert all of commits after Prism 0.19.0 release"

This reverts commit d242e84.

prism/config.yml

@@ -368,8 +368,8 @@ flags:
     comment: Flags for integer nodes that correspond to the base of the integer.
   - name: KeywordHashNodeFlags
     values:
-      - name: STATIC_KEYS
-        comment: "a keyword hash which only has `AssocNode` elements all with static literal keys, which means the elements can be treated as keyword arguments"
+      - name: SYMBOL_KEYS
+        comment: "a keyword hash which only has `AssocNode` elements all with symbol keys, which means the elements can be treated as keyword arguments"
     comment: Flags for keyword hash nodes.
   - name: LoopFlags
     values:

prism/diagnostic.c

@@ -62,6 +62,7 @@ static const char* const diagnostic_messages[PM_DIAGNOSTIC_ID_LEN] = {
     [PM_ERR_ARGUMENT_FORMAL_GLOBAL]             = "invalid formal argument; formal argument cannot be a global variable",
     [PM_ERR_ARGUMENT_FORMAL_IVAR]               = "invalid formal argument; formal argument cannot be an instance variable",
     [PM_ERR_ARGUMENT_FORWARDING_UNBOUND]        = "unexpected `...` in an non-parenthesized call",
+    [PM_ERR_ARGUMENT_IN]                        = "unexpected `in` keyword in arguments",
     [PM_ERR_ARGUMENT_NO_FORWARDING_AMP]         = "unexpected `&` when the parent method is not forwarding",
     [PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES]    = "unexpected `...` when the parent method is not forwarding",
     [PM_ERR_ARGUMENT_NO_FORWARDING_STAR]        = "unexpected `*` when the parent method is not forwarding",
@@ -191,6 +192,7 @@ static const char* const diagnostic_messages[PM_DIAGNOSTIC_ID_LEN] = {
     [PM_ERR_MODULE_TERM]                        = "expected an `end` to close the `module` statement",
     [PM_ERR_MULTI_ASSIGN_MULTI_SPLATS]          = "multiple splats in multiple assignment",
     [PM_ERR_NOT_EXPRESSION]                     = "expected an expression after `not`",
+    [PM_ERR_NO_LOCAL_VARIABLE]                  = "%.*s: no such local variable",
     [PM_ERR_NUMBER_LITERAL_UNDERSCORE]          = "number literal ending with a `_`",
     [PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED]     = "numbered parameters are not allowed when an ordinary parameter is defined",
     [PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE]     = "numbered parameter is already used in outer scope",

prism/diagnostic.h

@@ -53,6 +53,7 @@ typedef enum {
     PM_ERR_ARGUMENT_FORMAL_GLOBAL,
     PM_ERR_ARGUMENT_FORMAL_IVAR,
     PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
+    PM_ERR_ARGUMENT_IN,
     PM_ERR_ARGUMENT_NO_FORWARDING_AMP,
     PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
     PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
@@ -183,6 +184,7 @@ typedef enum {
     PM_ERR_MODULE_TERM,
     PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
     PM_ERR_NOT_EXPRESSION,
+    PM_ERR_NO_LOCAL_VARIABLE,
     PM_ERR_NUMBER_LITERAL_UNDERSCORE,
     PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
     PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,

prism/parser.h

@@ -17,12 +17,6 @@
 
 #include <stdbool.h>
 
-// TODO: remove this by renaming the original flag
-/**
- * Temporary alias for the PM_NODE_FLAG_STATIC_KEYS flag.
- */
-#define PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS PM_KEYWORD_HASH_NODE_FLAGS_STATIC_KEYS
-
 /**
  * This enum provides various bits that represent different kinds of states that
  * the lexer can track. This is used to determine which kind of token to return

prism/prism.c

@@ -1340,6 +1340,11 @@ pm_assoc_node_create(pm_parser_t *parser, pm_node_t *key, const pm_token_t *oper
         flags = key->flags & value->flags & PM_NODE_FLAG_STATIC_LITERAL;
     }
 
+    // Hash string keys should be frozen
+    if (PM_NODE_TYPE_P(key, PM_STRING_NODE)) {
+        key->flags |= PM_STRING_FLAGS_FROZEN;
+    }
+
     *node = (pm_assoc_node_t) {
         {
             .type = PM_ASSOC_NODE,
@@ -11338,6 +11343,9 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for
                     }
 
                     parsed_bare_hash = true;
+                } else if (accept1(parser, PM_TOKEN_KEYWORD_IN)) {
+                    // TODO: Could we solve this with binding powers instead?
+                    pm_parser_err_current(parser, PM_ERR_ARGUMENT_IN);
                 }
 
                 parse_arguments_append(parser, arguments, argument);
@@ -13349,8 +13357,15 @@ parse_pattern_primitive(pm_parser_t *parser, pm_diagnostic_id_t diag_id) {
             // expression to determine if it's a variable or an expression.
             switch (parser->current.type) {
                 case PM_TOKEN_IDENTIFIER: {
+                    int depth = pm_parser_local_depth(parser, &parser->current);
+
+                    if (depth == -1) {
+                        depth = 0;
+                        PM_PARSER_ERR_TOKEN_FORMAT(parser, parser->current, PM_ERR_NO_LOCAL_VARIABLE, (int) (parser->current.end - parser->current.start), parser->current.start);
+                    }
+
+                    pm_node_t *variable = (pm_node_t *) pm_local_variable_read_node_create(parser, &parser->current, (uint32_t) depth);
                     parser_lex(parser);
-                    pm_node_t *variable = (pm_node_t *) pm_local_variable_read_node_create(parser, &parser->previous, 0);
 
                     return (pm_node_t *) pm_pinned_variable_node_create(parser, &operator, variable);
                 }
@@ -17175,9 +17190,14 @@ parse_expression(pm_parser_t *parser, pm_binding_power_t binding_power, bool acc
 
 static pm_node_t *
 parse_program(pm_parser_t *parser) {
-    pm_parser_scope_push(parser, !parser->current_scope);
-    parser_lex(parser);
+    // If the current scope is NULL, then we want to push a new top level scope.
+    // The current scope could exist in the event that we are parsing an eval
+    // and the user has passed into scopes that already exist.
+    if (parser->current_scope == NULL) {
+        pm_parser_scope_push(parser, true);
+    }
 
+    parser_lex(parser);
     pm_statements_node_t *statements = parse_statements(parser, PM_CONTEXT_MAIN);
     if (!statements) {
         statements = pm_statements_node_create(parser);

prism/templates/lib/prism/node.rb.erb

@@ -44,19 +44,19 @@ module Prism
   <%- end -%>
   class <%= node.name -%> < Node
     <%- node.fields.each do |field| -%>
-    # attr_reader <%= field.name %>: <%= field.rbs_class %>
+    # <%= "private " if field.is_a?(Prism::FlagsField) %>attr_reader <%= field.name %>: <%= field.rbs_class %>
     <%= "private " if field.is_a?(Prism::FlagsField) %>attr_reader :<%= field.name %>
 
     <%- end -%>
-    # def initialize: (<%= (node.fields.map { |field| "#{field.name}: #{field.rbs_class}" } + ["location: Location"]).join(", ") %>) -> void
+    # def initialize: (<%= (node.fields.map { |field| "#{field.rbs_class} #{field.name}" } + ["Location location"]).join(", ") %>) -> void
     def initialize(<%= (node.fields.map(&:name) + ["location"]).join(", ") %>)
       <%- node.fields.each do |field| -%>
       @<%= field.name %> = <%= field.name %>
       <%- end -%>
       @location = location
     end
 
-    # def accept: (visitor: Visitor) -> void
+    # def accept: (Visitor visitor) -> void
     def accept(visitor)
       visitor.visit_<%= node.human %>(self)
     end
@@ -137,7 +137,7 @@ module Prism
     # def deconstruct: () -> Array[nil | Node]
     alias deconstruct child_nodes
 
-    # def deconstruct_keys: (keys: Array[Symbol]) -> Hash[Symbol, nil | Node | Array[Node] | String | Token | Array[Token] | Location]
+    # def deconstruct_keys: (Array[Symbol] keys) -> { <%= (node.fields.map { |field| "#{field.name}: #{field.rbs_class}" } + ["location: Location"]).join(", ") %> }
     def deconstruct_keys(keys)
       { <%= (node.fields.map { |field| "#{field.name}: #{field.name}" } + ["location: location"]).join(", ") %> }
     end
@@ -170,7 +170,7 @@ module Prism
     <%- end -%>
     <%- end -%>
 
-    # def inspect(inspector: NodeInspector) -> String
+    # def inspect(NodeInspector inspector) -> String
     def inspect(inspector = NodeInspector.new)
       inspector << inspector.header(self)
       <%- node.fields.each_with_index do |field, index| -%>

test/prism/errors_test.rb

@@ -1997,6 +1997,21 @@ def test_range_and_bin_op
       end
     end
 
+    def test_command_call_in
+      source = <<~RUBY
+        foo 1 in a
+        a = foo 2 in b
+      RUBY
+      message1 = 'unexpected `in` keyword in arguments'
+      message2 = 'expected a newline or semicolon after the statement'
+      assert_errors expression(source), source, [
+        [message1, 9..10],
+        [message2, 8..8],
+        [message1, 24..25],
+        [message2, 23..23],
+      ]
+    end
+
     def test_constant_assignment_in_method
       source = 'def foo();A=1;end'
       assert_errors expression(source), source, [

test/prism/fixtures/patterns.txt

@@ -51,7 +51,7 @@ foo => __LINE__ .. __LINE__
 foo => __ENCODING__ .. __ENCODING__
 foo => -> { bar } .. -> { bar }
 
-foo => ^bar
+bar = 1; foo => ^bar
 foo => ^@bar
 foo => ^@@bar
 foo => ^$bar

test/prism/location_test.rb

@@ -673,7 +673,7 @@ def test_PinnedExpressionNode
     end
 
     def test_PinnedVariableNode
-      assert_location(PinnedVariableNode, "foo in ^bar", 7...11, &:pattern)
+      assert_location(PinnedVariableNode, "bar = 1; foo in ^bar", 16...20, &:pattern)
     end
 
     def test_PostExecutionNode

test/prism/ruby_api_test.rb

@@ -42,7 +42,9 @@ def test_options
 
       assert_kind_of Prism::CallNode, Prism.parse("foo").value.statements.body[0]
       assert_kind_of Prism::LocalVariableReadNode, Prism.parse("foo", scopes: [[:foo]]).value.statements.body[0]
-      assert_equal 2, Prism.parse("foo", scopes: [[:foo], []]).value.statements.body[0].depth
+      assert_equal 1, Prism.parse("foo", scopes: [[:foo], []]).value.statements.body[0].depth
+
+      assert_equal [:foo], Prism.parse("foo", scopes: [[:foo]]).value.locals
     end
 
     def test_literal_value_method

test/prism/snapshots/arrays.txt

@@ -79,7 +79,7 @@
         │   ├── flags: ∅
         │   ├── elements: (length: 1)
         │   │   └── @ KeywordHashNode (location: (5,1)-(5,12))
-        │   │       ├── flags: static_keys
+        │   │       ├── flags: symbol_keys
         │   │       └── elements: (length: 1)
         │   │           └── @ AssocNode (location: (5,1)-(5,12))
         │   │               ├── key:

test/prism/snapshots/if.txt

@@ -250,7 +250,7 @@
         │   │           │   ├── flags: ∅
         │   │           │   └── arguments: (length: 1)
         │   │           │       └── @ KeywordHashNode (location: (25,4)-(25,6))
-        │   │           │           ├── flags: static_keys
+        │   │           │           ├── flags: symbol_keys
         │   │           │           └── elements: (length: 1)
         │   │           │               └── @ AssocNode (location: (25,4)-(25,6))
         │   │           │                   ├── key:

test/prism/snapshots/method_calls.txt

@@ -782,7 +782,7 @@
         │   │       │   ├── closing_loc: ∅
         │   │       │   └── unescaped: "a"
         │   │       └── @ KeywordHashNode (location: (60,8)-(60,32))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 2)
         │   │               ├── @ AssocNode (location: (60,8)-(60,22))
         │   │               │   ├── key:
@@ -914,7 +914,7 @@
         │   │       │   ├── closing_loc: ∅
         │   │       │   └── unescaped: "a"
         │   │       └── @ KeywordHashNode (location: (64,8)-(64,15))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (64,8)-(64,15))
         │   │                   ├── key:
@@ -983,7 +983,7 @@
         │   │   ├── flags: ∅
         │   │   └── arguments: (length: 1)
         │   │       └── @ KeywordHashNode (location: (66,3)-(66,17))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (66,3)-(66,17))
         │   │                   ├── key:
@@ -1173,7 +1173,7 @@
         │   │   ├── flags: ∅
         │   │   └── arguments: (length: 1)
         │   │       └── @ KeywordHashNode (location: (74,3)-(74,20))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (74,3)-(74,20))
         │   │                   ├── key:
@@ -1236,7 +1236,7 @@
         │   │       │   ├── closing_loc: ∅
         │   │       │   └── unescaped: "a"
         │   │       └── @ KeywordHashNode (location: (82,0)-(82,5))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (82,0)-(82,5))
         │   │                   ├── key:
@@ -1287,7 +1287,7 @@
         │   │   ├── flags: ∅
         │   │   └── arguments: (length: 1)
         │   │       └── @ KeywordHashNode (location: (87,4)-(87,21))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 2)
         │   │               ├── @ AssocNode (location: (87,4)-(87,11))
         │   │               │   ├── key:
@@ -1336,7 +1336,7 @@
         │   │       ├── @ IntegerNode (location: (89,10)-(89,11))
         │   │       │   └── flags: decimal
         │   │       └── @ KeywordHashNode (location: (89,13)-(89,21))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (89,13)-(89,21))
         │   │                   ├── key:
@@ -1527,7 +1527,7 @@
         │   │   ├── flags: ∅
         │   │   └── arguments: (length: 1)
         │   │       └── @ KeywordHashNode (location: (103,4)-(103,11))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (103,4)-(103,11))
         │   │                   ├── key:
@@ -1555,7 +1555,7 @@
         │   │   ├── flags: ∅
         │   │   └── arguments: (length: 1)
         │   │       └── @ KeywordHashNode (location: (105,4)-(105,28))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (105,4)-(105,28))
         │   │                   ├── key:
@@ -1612,7 +1612,7 @@
         │   │   ├── flags: ∅
         │   │   └── arguments: (length: 1)
         │   │       └── @ KeywordHashNode (location: (107,4)-(107,24))
-        │   │           ├── flags: static_keys
+        │   │           ├── flags: symbol_keys
         │   │           └── elements: (length: 1)
         │   │               └── @ AssocNode (location: (107,4)-(107,24))
         │   │                   ├── key:

test/prism/snapshots/patterns.txt

@@ -2,7 +2,7 @@
 ├── locals: [:bar, :baz, :qux, :b, :a, :foo, :x]
 └── statements:
     @ StatementsNode (location: (1,0)-(202,19))
-    └── body: (length: 175)
+    └── body: (length: 176)
         ├── @ MatchRequiredNode (location: (1,0)-(1,10))
         │   ├── value:
         │   │   @ CallNode (location: (1,0)-(1,3))
@@ -1235,26 +1235,34 @@
         │   │   │               └── depth: 1
         │   │   └── operator_loc: (52,18)-(52,20) = ".."
         │   └── operator_loc: (52,4)-(52,6) = "=>"
-        ├── @ MatchRequiredNode (location: (54,0)-(54,11))
+        ├── @ LocalVariableWriteNode (location: (54,0)-(54,7))
+        │   ├── name: :bar
+        │   ├── depth: 0
+        │   ├── name_loc: (54,0)-(54,3) = "bar"
         │   ├── value:
-        │   │   @ CallNode (location: (54,0)-(54,3))
+        │   │   @ IntegerNode (location: (54,6)-(54,7))
+        │   │   └── flags: decimal
+        │   └── operator_loc: (54,4)-(54,5) = "="
+        ├── @ MatchRequiredNode (location: (54,9)-(54,20))
+        │   ├── value:
+        │   │   @ CallNode (location: (54,9)-(54,12))
         │   │   ├── flags: variable_call
         │   │   ├── receiver: ∅
         │   │   ├── call_operator_loc: ∅
         │   │   ├── name: :foo
-        │   │   ├── message_loc: (54,0)-(54,3) = "foo"
+        │   │   ├── message_loc: (54,9)-(54,12) = "foo"
         │   │   ├── opening_loc: ∅
         │   │   ├── arguments: ∅
         │   │   ├── closing_loc: ∅
         │   │   └── block: ∅
         │   ├── pattern:
-        │   │   @ PinnedVariableNode (location: (54,7)-(54,11))
+        │   │   @ PinnedVariableNode (location: (54,16)-(54,20))
         │   │   ├── variable:
-        │   │   │   @ LocalVariableReadNode (location: (54,8)-(54,11))
+        │   │   │   @ LocalVariableReadNode (location: (54,17)-(54,20))
         │   │   │   ├── name: :bar
         │   │   │   └── depth: 0
-        │   │   └── operator_loc: (54,7)-(54,8) = "^"
-        │   └── operator_loc: (54,4)-(54,6) = "=>"
+        │   │   └── operator_loc: (54,16)-(54,17) = "^"
+        │   └── operator_loc: (54,13)-(54,15) = "=>"
         ├── @ MatchRequiredNode (location: (55,0)-(55,12))
         │   ├── value:
         │   │   @ CallNode (location: (55,0)-(55,3))

test/prism/snapshots/rescue.txt

@@ -343,7 +343,7 @@
             │   │           │   ├── flags: ∅
             │   │           │   └── arguments: (length: 1)
             │   │           │       └── @ KeywordHashNode (location: (29,4)-(29,6))
-            │   │           │           ├── flags: static_keys
+            │   │           │           ├── flags: symbol_keys
             │   │           │           └── elements: (length: 1)
             │   │           │               └── @ AssocNode (location: (29,4)-(29,6))
             │   │           │                   ├── key:

test/prism/snapshots/seattlerb/assoc_label.txt

@@ -15,7 +15,7 @@
             │   ├── flags: ∅
             │   └── arguments: (length: 1)
             │       └── @ KeywordHashNode (location: (1,2)-(1,5))
-            │           ├── flags: static_keys
+            │           ├── flags: symbol_keys
             │           └── elements: (length: 1)
             │               └── @ AssocNode (location: (1,2)-(1,5))
             │                   ├── key:

test/prism/snapshots/seattlerb/bug_249.txt

@@ -66,7 +66,7 @@
             │       │   ├── closing_loc: ∅
             │       │   └── block: ∅
             │       └── @ KeywordHashNode (location: (4,11)-(4,28))
-            │           ├── flags: static_keys
+            │           ├── flags: symbol_keys
             │           └── elements: (length: 1)
             │               └── @ AssocNode (location: (4,11)-(4,28))
             │                   ├── key: