Introduce partial_script option

Author: kddnewton

ext/prism/extension.c

@@ -32,6 +32,7 @@ ID rb_id_option_filepath;
 ID rb_id_option_frozen_string_literal;
 ID rb_id_option_line;
 ID rb_id_option_main_script;
+ID rb_id_option_partial_script;
 ID rb_id_option_scopes;
 ID rb_id_option_version;
 ID rb_id_source_for;
@@ -182,6 +183,8 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
         }
     } else if (key_id == rb_id_option_main_script) {
         if (!NIL_P(value)) pm_options_main_script_set(options, RTEST(value));
+    } else if (key_id == rb_id_option_partial_script) {
+        if (!NIL_P(value)) pm_options_partial_script_set(options, RTEST(value));
     } else {
         rb_raise(rb_eArgError, "unknown keyword: %" PRIsVALUE, key);
     }
@@ -761,6 +764,12 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
  *       or not shebangs are parsed for additional flags and whether or not the
  *       parser will attempt to find a matching shebang if the first one does
  *       not contain the word "ruby".
+ * * `partial_script` - when the file being parsed is considered a "partial"
+ *       script, jumps will not be marked as errors if they are not contained
+ *       within loops/blocks. This is used in the case that you're parsing a
+ *       script that you know will be embedded inside another script later, but
+ *       you do not have that context yet. For example, when parsing an ERB
+ *       template that will be evaluated inside another script.
  * * `scopes` - the locals that are in scope surrounding the code that is being
  *       parsed. This should be an array of arrays of symbols or nil. Scopes are
  *       ordered from the outermost scope to the innermost one.
@@ -1174,6 +1183,7 @@ Init_prism(void) {
     rb_id_option_frozen_string_literal = rb_intern_const("frozen_string_literal");
     rb_id_option_line = rb_intern_const("line");
     rb_id_option_main_script = rb_intern_const("main_script");
+    rb_id_option_partial_script = rb_intern_const("partial_script");
     rb_id_option_scopes = rb_intern_const("scopes");
     rb_id_option_version = rb_intern_const("version");
     rb_id_source_for = rb_intern("for");

include/prism/options.h

@@ -146,6 +146,16 @@ typedef struct pm_options {
      * to pass this information to the parser so that it can behave correctly.
      */
     bool main_script;
+
+    /**
+     * When the file being parsed is considered a "partial" script, jumps will
+     * not be marked as errors if they are not contained within loops/blocks.
+     * This is used in the case that you're parsing a script that you know will
+     * be embedded inside another script later, but you do not have that context
+     * yet. For example, when parsing an ERB template that will be evaluated
+     * inside another script.
+     */
+    bool partial_script;
 } pm_options_t;
 
 /**
@@ -263,6 +273,14 @@ PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const
  */
 PRISM_EXPORTED_FUNCTION void pm_options_main_script_set(pm_options_t *options, bool main_script);
 
+/**
+ * Set the partial script option on the given options struct.
+ *
+ * @param options The options struct to set the partial script value on.
+ * @param partial_script The partial script value to set.
+ */
+PRISM_EXPORTED_FUNCTION void pm_options_partial_script_set(pm_options_t *options, bool partial_script);
+
 /**
  * Allocate and zero out the scopes array on the given options struct.
  *
@@ -330,6 +348,9 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | `1`     | -l command line option     |
  * | `1`     | -a command line option     |
  * | `1`     | the version                |
+ * | `1`     | encoding locked            |
+ * | `1`     | main script                |
+ * | `1`     | partial script             |
  * | `4`     | the number of scopes       |
  * | ...     | the scopes                 |
  *
@@ -362,8 +383,8 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * * The encoding can have a length of 0, in which case we'll use the default
  *   encoding (UTF-8). If it's not 0, it should correspond to a name of an
  *   encoding that can be passed to `Encoding.find` in Ruby.
- * * The frozen string literal and suppress warnings fields are booleans, so
- *   their values should be either 0 or 1.
+ * * The frozen string literal, encoding locked, main script, and partial script
+ *   fields are booleans, so their values should be either 0 or 1.
  * * The number of scopes can be 0.
  *
  * @param options The options struct to deserialize into.

include/prism/parser.h

@@ -861,6 +861,13 @@ struct pm_parser {
      */
     bool parsing_eval;
 
+    /**
+     * Whether or not we are parsing a "partial" script, which is a script that
+     * will be evaluated in the context of another script, so we should not
+     * check jumps (next/break/etc.) for validity.
+     */
+    bool partial_script;
+
     /** Whether or not we're at the beginning of a command. */
     bool command_start;
 

java-wasm/src/test/java/org/prism/DummyTest.java

@@ -42,6 +42,7 @@ public void test1() {
             ParsingOptions.SyntaxVersion.LATEST,
             false,
             false,
+            false,
             new byte[][][] {}
         );
 
@@ -95,6 +96,7 @@ public void test2() {
             ParsingOptions.SyntaxVersion.LATEST,
             false,
             false,
+            false,
             new byte[][][] {}
         );
 

java/org/prism/ParsingOptions.java

@@ -45,10 +45,11 @@ public enum CommandLine { A, E, L, N, P, X };
      * @param version code of Ruby version which syntax will be used to parse
      * @param encodingLocked whether the encoding is locked (should almost always be false)
      * @param mainScript whether the file is the main script
+     * @param partialScript whether the file is a partial script
      * @param scopes scopes surrounding the code that is being parsed with local variable names defined in every scope
      *            ordered from the outermost scope to the innermost one
      */
-    public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boolean frozenStringLiteral, EnumSet<CommandLine> commandLine, SyntaxVersion version, boolean encodingLocked, boolean mainScript, byte[][][] scopes) {
+    public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boolean frozenStringLiteral, EnumSet<CommandLine> commandLine, SyntaxVersion version, boolean encodingLocked, boolean mainScript, boolean partialScript, byte[][][] scopes) {
         final ByteArrayOutputStream output = new ByteArrayOutputStream();
 
         // filepath
@@ -77,6 +78,9 @@ public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boole
         // mainScript
         output.write(mainScript ? 1 : 0);
 
+        // partialScript
+        output.write(partialScript ? 1 : 0);
+
         // scopes
 
         // number of scopes

javascript/src/parsePrism.js

@@ -62,6 +62,11 @@ function dumpCommandLineOptions(options) {
   return value;
 }
 
+// Convert a boolean value into a serialized byte.
+function dumpBooleanOption(value) {
+  return (value === undefined || value === false || value === null) ? 0 : 1;
+}
+
 // Converts the given options into a serialized options string.
 function dumpOptions(options) {
   const values = [];
@@ -92,7 +97,7 @@ function dumpOptions(options) {
   }
 
   template.push("C");
-  values.push((options.frozen_string_literal === undefined || options.frozen_string_literal === false || options.frozen_string_literal === null) ? 0 : 1);
+  values.push(dumpBooleanOption(options.frozen_string_literal));
 
   template.push("C");
   values.push(dumpCommandLineOptions(options));
@@ -110,7 +115,10 @@ function dumpOptions(options) {
   values.push(options.encoding === false ? 1 : 0);
 
   template.push("C");
-  values.push((options.main_script === undefined || options.main_script === false || options.main_script === null) ? 0 : 1);
+  values.push(dumpBooleanOption(options.main_script));
+
+  template.push("C");
+  values.push(dumpBooleanOption(options.partial_script));
 
   template.push("L");
   if (options.scopes) {

lib/prism/ffi.rb

@@ -451,6 +451,9 @@ def dump_options(options)
       template << "C"
       values << (options.fetch(:main_script, false) ? 1 : 0)
 
+      template << "C"
+      values << (options.fetch(:partial_script, false) ? 1 : 0)
+
       template << "L"
       if (scopes = options[:scopes])
         values << scopes.length

src/options.c

@@ -108,6 +108,14 @@ pm_options_main_script_set(pm_options_t *options, bool main_script) {
     options->main_script = main_script;
 }
 
+/**
+ * Set the partial script option on the given options struct.
+ */
+PRISM_EXPORTED_FUNCTION void
+pm_options_partial_script_set(pm_options_t *options, bool partial_script) {
+    options->partial_script = partial_script;
+}
+
 // For some reason, GCC analyzer thinks we're leaking allocated scopes and
 // locals here, even though we definitely aren't. This is a false positive.
 // Ideally we wouldn't need to suppress this.
@@ -242,6 +250,7 @@ pm_options_read(pm_options_t *options, const char *data) {
     options->version = (pm_options_version_t) *data++;
     options->encoding_locked = ((uint8_t) *data++) > 0;
     options->main_script = ((uint8_t) *data++) > 0;
+    options->partial_script = ((uint8_t) *data++) > 0;
 
     uint32_t scopes_count = pm_options_read_u32(data);
     data += 4;

src/prism.c

@@ -18852,12 +18852,12 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
             switch (keyword.type) {
                 case PM_TOKEN_KEYWORD_BREAK: {
                     pm_node_t *node = (pm_node_t *) pm_break_node_create(parser, &keyword, arguments.arguments);
-                    parse_block_exit(parser, node);
+                    if (!parser->partial_script) parse_block_exit(parser, node);
                     return node;
                 }
                 case PM_TOKEN_KEYWORD_NEXT: {
                     pm_node_t *node = (pm_node_t *) pm_next_node_create(parser, &keyword, arguments.arguments);
-                    parse_block_exit(parser, node);
+                    if (!parser->partial_script) parse_block_exit(parser, node);
                     return node;
                 }
                 case PM_TOKEN_KEYWORD_RETURN: {
@@ -18905,7 +18905,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
             }
 
             pm_node_t *node = (pm_node_t *) pm_yield_node_create(parser, &keyword, &arguments.opening_loc, arguments.arguments, &arguments.closing_loc);
-            if (!parser->parsing_eval) parse_yield(parser, node);
+            if (!parser->parsing_eval && !parser->partial_script) parse_yield(parser, node);
 
             return node;
         }
@@ -19574,7 +19574,7 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
             parser_lex(parser);
 
             pm_node_t *node = (pm_node_t *) pm_redo_node_create(parser, &parser->previous);
-            parse_block_exit(parser, node);
+            if (!parser->partial_script) parse_block_exit(parser, node);
 
             return node;
         }
@@ -21899,6 +21899,7 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm
         .explicit_encoding = NULL,
         .command_line = 0,
         .parsing_eval = false,
+        .partial_script = false,
         .command_start = true,
         .recovering = false,
         .encoding_locked = false,
@@ -21962,6 +21963,9 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm
         // version option
         parser->version = options->version;
 
+        // partial_script
+        parser->partial_script = options->partial_script;
+
         // scopes option
         parser->parsing_eval = options->scopes_count > 0;
         if (parser->parsing_eval) parser->warn_mismatched_indentation = false;

test/prism/api/parse_test.rb

@@ -90,6 +90,20 @@ def test_parse_directory
       assert_kind_of Errno::EISDIR, error
     end
 
+    def test_partial_script
+      assert Prism.parse_failure?("break")
+      assert Prism.parse_success?("break", partial_script: true)
+
+      assert Prism.parse_failure?("next")
+      assert Prism.parse_success?("next", partial_script: true)
+
+      assert Prism.parse_failure?("redo")
+      assert Prism.parse_success?("redo", partial_script: true)
+
+      assert Prism.parse_failure?("yield")
+      assert Prism.parse_success?("yield", partial_script: true)
+    end
+
     private
 
     def find_source_file_node(program)