Support forwarding flags on scopes

When parent scopes around an eval are forwarding parameters (like
*, **, &, or ...) we need to know that information when we are in
the parser. As such, we need to support passing that information
into the scopes option. In order to do this, unfortunately we need
a bunch of changes.

The scopes option was previously an array of array of strings.
These corresponded to the names of the locals in the parent scopes.
We still support this, but now additionally support passing in a
Prism::Scope instance at each index in the array. This Prism::Scope
class holds both the names of the locals as well as an array of
forwarding parameter names (symbols corresponding to the forwarding
parameters). There is convenience function on the Prism module that
creates a Prism::Scope object using Prism.scope.

In JavaScript, we now additionally support an object much the same
as the Ruby side. In Java, we now have a ParsingOptions.Scope class
that holds that information. In the dump APIs, these objects in all
3 languages will add an additional byte for the forwarding flags in
the middle of the scopes serialization.

All of this is in service of properly parsing the following code:

```ruby
def foo(*) = eval("bar(*)")
```

Author: kddnewton

ext/prism/extension.c

@@ -24,6 +24,7 @@ VALUE rb_cPrismParseResult;
 VALUE rb_cPrismLexResult;
 VALUE rb_cPrismParseLexResult;
 VALUE rb_cPrismStringQuery;
+VALUE rb_cPrismScope;
 
 VALUE rb_cPrismDebugEncoding;
 
@@ -38,6 +39,10 @@ ID rb_id_option_partial_script;
 ID rb_id_option_scopes;
 ID rb_id_option_version;
 ID rb_id_source_for;
+ID rb_id_forwarding_positionals;
+ID rb_id_forwarding_keywords;
+ID rb_id_forwarding_block;
+ID rb_id_forwarding_all;
 
 /******************************************************************************/
 /* IO of Ruby code                                                            */
@@ -95,22 +100,61 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
     for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
         VALUE scope = rb_ary_entry(scopes, scope_index);
 
-        // Check that the scope is an array. If it's not, then raise a type
-        // error.
-        if (!RB_TYPE_P(scope, T_ARRAY)) {
-            rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array)", rb_obj_class(scope));
+        // The scope can be either an array or it can be a Prism::Scope object.
+        // Parse out the correct values here from either.
+        VALUE locals;
+        uint8_t forwarding = PM_OPTIONS_SCOPE_FORWARDING_NONE;
+
+        if (RB_TYPE_P(scope, T_ARRAY)) {
+            locals = scope;
+        } else if (rb_obj_is_kind_of(scope, rb_cPrismScope)) {
+            locals = rb_ivar_get(scope, rb_intern("@locals"));
+            if (!RB_TYPE_P(locals, T_ARRAY)) {
+                rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array)", rb_obj_class(locals));
+            }
+
+            VALUE names = rb_ivar_get(scope, rb_intern("@forwarding"));
+            if (!RB_TYPE_P(names, T_ARRAY)) {
+                rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array)", rb_obj_class(names));
+            }
+
+            size_t names_count = RARRAY_LEN(names);
+            for (size_t name_index = 0; name_index < names_count; name_index++) {
+                VALUE name = rb_ary_entry(names, name_index);
+
+                // Check that the name is a symbol. If it's not, then raise
+                // a type error.
+                if (!RB_TYPE_P(name, T_SYMBOL)) {
+                    rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Symbol)", rb_obj_class(name));
+                }
+
+                ID id = SYM2ID(name);
+                if (id == rb_id_forwarding_positionals) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_POSITIONALS;
+                } else if (id == rb_id_forwarding_keywords) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_KEYWORDS;
+                } else if (id == rb_id_forwarding_block) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_BLOCK;
+                } else if (id == rb_id_forwarding_all) {
+                    forwarding |= PM_OPTIONS_SCOPE_FORWARDING_ALL;
+                } else {
+                    rb_raise(rb_eArgError, "invalid forwarding value: %" PRIsVALUE, name);
+                }
+            }
+        } else {
+            rb_raise(rb_eTypeError, "wrong argument type %"PRIsVALUE" (expected Array or Prism::Scope)", rb_obj_class(scope));
         }
 
         // Initialize the scope array.
-        size_t locals_count = RARRAY_LEN(scope);
+        size_t locals_count = RARRAY_LEN(locals);
         pm_options_scope_t *options_scope = &options->scopes[scope_index];
         if (!pm_options_scope_init(options_scope, locals_count)) {
             rb_raise(rb_eNoMemError, "failed to allocate memory");
         }
 
         // Iterate over the locals and add them to the scope.
         for (size_t local_index = 0; local_index < locals_count; local_index++) {
-            VALUE local = rb_ary_entry(scope, local_index);
+            VALUE local = rb_ary_entry(locals, local_index);
 
             // Check that the local is a symbol. If it's not, then raise a
             // type error.
@@ -123,6 +167,9 @@ build_options_scopes(pm_options_t *options, VALUE scopes) {
             const char *name = rb_id2name(SYM2ID(local));
             pm_string_constant_init(scope_local, name, strlen(name));
         }
+
+        // Now set the forwarding options.
+        pm_options_scope_forwarding_set(options_scope, forwarding);
     }
 }
 
@@ -1302,6 +1349,7 @@ Init_prism(void) {
     rb_cPrismLexResult = rb_define_class_under(rb_cPrism, "LexResult", rb_cPrismResult);
     rb_cPrismParseLexResult = rb_define_class_under(rb_cPrism, "ParseLexResult", rb_cPrismResult);
     rb_cPrismStringQuery = rb_define_class_under(rb_cPrism, "StringQuery", rb_cObject);
+    rb_cPrismScope = rb_define_class_under(rb_cPrism, "Scope", rb_cObject);
 
     // Intern all of the IDs eagerly that we support so that we don't have to do
     // it every time we parse.
@@ -1316,6 +1364,10 @@ Init_prism(void) {
     rb_id_option_scopes = rb_intern_const("scopes");
     rb_id_option_version = rb_intern_const("version");
     rb_id_source_for = rb_intern("for");
+    rb_id_forwarding_positionals = rb_intern("*");
+    rb_id_forwarding_keywords = rb_intern("**");
+    rb_id_forwarding_block = rb_intern("&");
+    rb_id_forwarding_all = rb_intern("...");
 
     /**
      * The version of the prism library.

include/prism/options.h

@@ -39,8 +39,26 @@ typedef struct pm_options_scope {
 
     /** The names of the locals in the scope. */
     pm_string_t *locals;
+
+    /** Flags for the set of forwarding parameters in this scope. */
+    uint8_t forwarding;
 } pm_options_scope_t;
 
+/** The default value for parameters. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_NONE = 0x0;
+
+/** When the scope is fowarding with the * parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_POSITIONALS = 0x1;
+
+/** When the scope is fowarding with the ** parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_KEYWORDS = 0x2;
+
+/** When the scope is fowarding with the & parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_BLOCK = 0x4;
+
+/** When the scope is fowarding with the ... parameter. */
+static const uint8_t PM_OPTIONS_SCOPE_FORWARDING_ALL = 0x8;
+
 // Forward declaration needed by the callback typedef.
 struct pm_options;
 
@@ -337,6 +355,14 @@ PRISM_EXPORTED_FUNCTION bool pm_options_scope_init(pm_options_scope_t *scope, si
  */
 PRISM_EXPORTED_FUNCTION const pm_string_t * pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index);
 
+/**
+ * Set the forwarding option on the given scope struct.
+ *
+ * @param scope The scope struct to set the forwarding on.
+ * @param forwarding The forwarding value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_scope_forwarding_set(pm_options_scope_t *scope, uint8_t forwarding);
+
 /**
  * Free the internal memory associated with the options.
  *
@@ -386,6 +412,7 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | # bytes | field                      |
  * | ------- | -------------------------- |
  * | `4`     | the number of locals       |
+ * | `1`     | the forwarding flags       |
  * | ...     | the locals                 |
  *
  * Each local is laid out as follows:

java/org/prism/ParsingOptions.java

@@ -35,6 +35,69 @@ public byte getValue() {
      */
     public enum CommandLine { A, E, L, N, P, X };
 
+    /**
+     * The forwarding options for a given scope in the parser.
+     */
+    public enum Forwarding {
+        NONE(0),
+        POSITIONAL(1),
+        KEYWORD(2),
+        BLOCK(4),
+        ALL(8);
+
+        private final int value;
+
+        Forwarding(int value) {
+            this.value = value;
+        }
+
+        public byte getValue() {
+            return (byte) value;
+        }
+    };
+
+    /**
+     * Represents a scope in the parser.
+     */
+    public static class Scope {
+        private byte[][] locals;
+        private Forwarding[] forwarding;
+
+        Scope(byte[][] locals) {
+            this(locals, new Forwarding[0]);
+        }
+
+        Scope(Forwarding[] forwarding) {
+            this(new byte[0][], forwarding);
+        }
+
+        Scope(byte[][] locals, Forwarding[] forwarding) {
+            this.locals = locals;
+            this.forwarding = forwarding;
+        }
+
+        public byte[][] getLocals() {
+            return locals;
+        }
+
+        public int getForwarding() {
+            int value = 0;
+            for (Forwarding f : forwarding) {
+                value |= f.getValue();
+            }
+            return value;
+        }
+    }
+
+    public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boolean frozenStringLiteral, EnumSet<CommandLine> commandLine, SyntaxVersion version, boolean encodingLocked, boolean mainScript, boolean partialScript, byte[][][] scopes) {
+        Scope[] normalizedScopes = new Scope[scopes.length];
+        for (int i = 0; i < scopes.length; i++) {
+            normalizedScopes[i] = new Scope(scopes[i]);
+        }
+
+        return serialize(filepath, line, encoding, frozenStringLiteral, commandLine, version, encodingLocked, mainScript, partialScript, normalizedScopes);
+    }
+
     /**
      * Serialize parsing options into byte array.
      *
@@ -50,7 +113,7 @@ public enum CommandLine { A, E, L, N, P, X };
      * @param scopes scopes surrounding the code that is being parsed with local variable names defined in every scope
      *            ordered from the outermost scope to the innermost one
      */
-    public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boolean frozenStringLiteral, EnumSet<CommandLine> commandLine, SyntaxVersion version, boolean encodingLocked, boolean mainScript, boolean partialScript, byte[][][] scopes) {
+    public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boolean frozenStringLiteral, EnumSet<CommandLine> commandLine, SyntaxVersion version, boolean encodingLocked, boolean mainScript, boolean partialScript, Scope[] scopes) {
         final ByteArrayOutputStream output = new ByteArrayOutputStream();
 
         // filepath
@@ -91,12 +154,17 @@ public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boole
         write(output, serializeInt(scopes.length));
 
         // local variables in each scope
-        for (byte[][] scope : scopes) {
+        for (Scope scope : scopes) {
+            byte[][] locals = scope.getLocals();
+
             // number of locals
-            write(output, serializeInt(scope.length));
+            write(output, serializeInt(locals.length));
+
+            // forwarding flags
+            output.write(scope.getForwarding());
 
             // locals
-            for (byte[] local : scope) {
+            for (byte[] local : locals) {
                 write(output, serializeInt(local.length));
                 write(output, local);
             }

javascript/src/index.js

@@ -11,7 +11,9 @@ export * from "./nodes.js";
 /**
  * Load the prism wasm module and return a parse function.
  *
- * @returns {Promise<(source: string) => ParseResult>}
+ * @typedef {import("./parsePrism.js").Options} Options
+ *
+ * @returns {Promise<(source: string, options?: Options) => ParseResult>}
  */
 export async function loadPrism() {
   const wasm = await WebAssembly.compile(await readFile(fileURLToPath(new URL("prism.wasm", import.meta.url))));
@@ -20,7 +22,7 @@ export async function loadPrism() {
   const instance = await WebAssembly.instantiate(wasm, wasi.getImportObject());
   wasi.initialize(instance);
 
-  return function (source) {
-    return parsePrism(instance.exports, source);
+  return function (source, options = {}) {
+    return parsePrism(instance.exports, source, options);
   }
 }