CLI -x flag

kddnewton · kddnewton · commit 2068e3c30aca · 2024-03-28T10:02:10.000-04:00
diff --git a/config.yml b/config.yml
@@ -206,6 +206,7 @@ errors:
   - RESCUE_TERM
   - RESCUE_VARIABLE
   - RETURN_INVALID
+  - SCRIPT_NOT_FOUND
   - SINGLETON_FOR_LITERALS
   - STATEMENT_ALIAS
   - STATEMENT_POSTEXE_END
@@ -255,6 +256,7 @@ warnings:
   - KEYWORD_EOL
   - LITERAL_IN_CONDITION_DEFAULT
   - LITERAL_IN_CONDITION_VERBOSE
+  - SHEBANG_CARRIAGE_RETURN
   - UNEXPECTED_CARRIAGE_RETURN
 tokens:
   - name: EOF
diff --git a/docs/serialization.md b/docs/serialization.md
@@ -58,7 +58,7 @@ The comment type is one of:
 | `1` | type |
 | string | error message (ASCII-only characters) |
 | location | the location in the source this error applies to |
-| `1` | the level of the error: `0` for `fatal` |
+| `1` | the level of the error: `0` for `fatal`, `1` for `argument`, `2` for `load` |
 
 ### warning
 
@@ -194,7 +194,6 @@ The final argument to `pm_serialize_parse` is an optional string that controls t
 | `4`     | the length of the filepath |
 | ...     | the filepath bytes         |
 | `4`     | the line number            |
-| `4`     | the offset in the source   |
 | `4`     | the length the encoding    |
 | ...     | the encoding bytes         |
 | `1`     | frozen string literal      |
diff --git a/ext/prism/extension.c b/ext/prism/extension.c
@@ -28,7 +28,6 @@ ID rb_option_id_encoding;
 ID rb_option_id_filepath;
 ID rb_option_id_frozen_string_literal;
 ID rb_option_id_line;
-ID rb_option_id_offset;
 ID rb_option_id_scopes;
 ID rb_option_id_version;
 
@@ -139,8 +138,6 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
         if (!NIL_P(value)) pm_options_encoding_set(options, rb_enc_name(rb_to_encoding(value)));
     } else if (key_id == rb_option_id_line) {
         if (!NIL_P(value)) pm_options_line_set(options, NUM2INT(value));
-    } else if (key_id == rb_option_id_offset) {
-        if (!NIL_P(value)) pm_options_offset_set(options, NUM2UINT(value));
     } else if (key_id == rb_option_id_frozen_string_literal) {
         if (!NIL_P(value)) pm_options_frozen_string_literal_set(options, RTEST(value));
     } else if (key_id == rb_option_id_version) {
@@ -448,8 +445,8 @@ parser_errors(pm_parser_t *parser, rb_encoding *encoding, VALUE source) {
 
         VALUE level = Qnil;
         switch (error->level) {
-            case PM_ERROR_LEVEL_FATAL:
-                level = ID2SYM(rb_intern("fatal"));
+            case PM_ERROR_LEVEL_SYNTAX:
+                level = ID2SYM(rb_intern("syntax"));
                 break;
             case PM_ERROR_LEVEL_ARGUMENT:
                 level = ID2SYM(rb_intern("argument"));
@@ -1347,7 +1344,6 @@ Init_prism(void) {
     rb_option_id_filepath = rb_intern_const("filepath");
     rb_option_id_frozen_string_literal = rb_intern_const("frozen_string_literal");
     rb_option_id_line = rb_intern_const("line");
-    rb_option_id_offset = rb_intern_const("offset");
     rb_option_id_scopes = rb_intern_const("scopes");
     rb_option_id_version = rb_intern_const("version");
 
diff --git a/include/prism/options.h b/include/prism/options.h
@@ -66,12 +66,6 @@ typedef struct {
      */
     int32_t line;
 
-    /**
-     * The offset within the file that the parse starts on. This value is
-     * 0-indexed.
-     */
-    uint32_t offset;
-
     /**
      * The name of the encoding that the source file is in. Note that this must
      * correspond to a name that can be found with Encoding.find in Ruby.
@@ -164,14 +158,6 @@ PRISM_EXPORTED_FUNCTION void pm_options_filepath_set(pm_options_t *options, cons
  */
 PRISM_EXPORTED_FUNCTION void pm_options_line_set(pm_options_t *options, int32_t line);
 
-/**
- * Set the offset option on the given options struct.
- *
- * @param options The options struct to set the offset on.
- * @param offset The offset to set.
- */
-PRISM_EXPORTED_FUNCTION void pm_options_offset_set(pm_options_t *options, uint32_t offset);
-
 /**
  * Set the encoding option on the given options struct.
  *
@@ -267,7 +253,6 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
  * | `4`     | the length of the filepath |
  * | ...     | the filepath bytes         |
  * | `4`     | the line number            |
- * | `4`     | the offset                 |
  * | `4`     | the length the encoding    |
  * | ...     | the encoding bytes         |
  * | `1`     | frozen string literal      |
diff --git a/include/prism/util/pm_newline_list.h b/include/prism/util/pm_newline_list.h
@@ -61,6 +61,12 @@ typedef struct {
  */
 bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capacity);
 
+/**
+ * Clear out the newlines that have been appended to the list.
+ */
+void
+pm_newline_list_clear(pm_newline_list_t *list);
+
 /**
  * Append a new offset to the newline list. Returns true if the reallocation of
  * the offsets succeeds (if one was necessary), otherwise returns false.
diff --git a/java-wasm/src/test/java/org/prism/DummyTest.java b/java-wasm/src/test/java/org/prism/DummyTest.java
@@ -36,7 +36,6 @@ public void test1() {
         var packedOptions = ParsingOptions.serialize(
             new byte[] {},
             1,
-            0,
             new byte[] {},
             false,
             EnumSet.noneOf(ParsingOptions.CommandLine.class),
@@ -88,7 +87,6 @@ public void test2() {
         var packedOptions = ParsingOptions.serialize(
             new byte[] {},
             1,
-            0,
             new byte[] {},
             false,
             EnumSet.noneOf(ParsingOptions.CommandLine.class),
diff --git a/java/org/prism/ParsingOptions.java b/java/org/prism/ParsingOptions.java
@@ -39,15 +39,14 @@ public enum CommandLine { A, E, L, N, P, X };
      *
      * @param filepath the name of the file that is currently being parsed
      * @param line the line within the file that the parser starts on. This value is 1-indexed
-     * @param offset the offset within the file that the parser starts on. This value is 0-indexed
      * @param encoding the name of the encoding that the source file is in
      * @param frozenStringLiteral whether the frozen string literal option has been set
      * @param commandLine the set of flags that were set on the command line
      * @param version code of Ruby version which syntax will be used to parse
      * @param scopes scopes surrounding the code that is being parsed with local variable names defined in every scope
      *            ordered from the outermost scope to the innermost one
      */
-    public static byte[] serialize(byte[] filepath, int line, int offset, byte[] encoding, boolean frozenStringLiteral, EnumSet<CommandLine> commandLine, SyntaxVersion version, byte[][][] scopes) {
+    public static byte[] serialize(byte[] filepath, int line, byte[] encoding, boolean frozenStringLiteral, EnumSet<CommandLine> commandLine, SyntaxVersion version, byte[][][] scopes) {
         final ByteArrayOutputStream output = new ByteArrayOutputStream();
 
         // filepath
@@ -57,9 +56,6 @@ public static byte[] serialize(byte[] filepath, int line, int offset, byte[] enc
         // line
         write(output, serializeInt(line));
 
-        // offset
-        write(output, serializeInt(offset));
-
         // encoding
         write(output, serializeInt(encoding.length));
         write(output, encoding);
diff --git a/javascript/src/parsePrism.js b/javascript/src/parsePrism.js
@@ -81,9 +81,6 @@ function dumpOptions(options) {
   template.push("l");
   values.push(options.line || 1);
 
-  template.push("L");
-  values.push(options.offset || 0);
-
   template.push("L");
   if (options.encoding) {
     const encoding = encoder.encode(options.encoding);
diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb
@@ -382,9 +382,6 @@ def dump_options(options)
       template << "l"
       values << options.fetch(:line, 1)
 
-      template << "L"
-      values << options.fetch(:offset, 0)
-
       template << "L"
       if (encoding = options[:encoding])
         name = encoding.name
diff --git a/src/options.c b/src/options.c
@@ -24,14 +24,6 @@ pm_options_line_set(pm_options_t *options, int32_t line) {
     options->line = line;
 }
 
-/**
- * Set the offset option on the given options struct.
- */
-PRISM_EXPORTED_FUNCTION void
-pm_options_offset_set(pm_options_t *options, uint32_t offset) {
-    options->offset = offset;
-}
-
 /**
  * Set the frozen string literal option on the given options struct.
  */
@@ -201,9 +193,6 @@ pm_options_read(pm_options_t *options, const char *data) {
     options->line = pm_options_read_s32(data);
     data += 4;
 
-    options->offset = pm_options_read_u32(data);
-    data += 4;
-
     uint32_t encoding_length = pm_options_read_u32(data);
     data += 4;
 
diff --git a/src/prism.c b/src/prism.c
@@ -500,6 +500,9 @@ debug_lex_state_set(pm_parser_t *parser, pm_lex_state_t state, char const * call
 /** True if the -p command line option was given. */
 #define PM_PARSER_COMMAND_LINE_OPTION_P(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_P)
 
+/** True if the -x command line option was given. */
+#define PM_PARSER_COMMAND_LINE_OPTION_X(parser) PM_PARSER_COMMAND_LINE_OPTION(parser, PM_OPTIONS_COMMAND_LINE_X)
+
 /******************************************************************************/
 /* Diagnostic-related functions                                               */
 /******************************************************************************/
@@ -19122,6 +19125,38 @@ parse_program(pm_parser_t *parser) {
 /* External functions                                                         */
 /******************************************************************************/
 
+/**
+ * A vendored version of strnstr that is used to find a substring within a
+ * string with a given length. This function is used to search for the Ruby
+ * engine name within a shebang when the -x option is passed to Ruby.
+ *
+ * The only modification that we made here is that we don't do NULL byte checks
+ * because we know the little parameter will not have a NULL byte and we allow
+ * the big parameter to have them.
+ */
+static const char *
+pm_strnstr(const char *big, const char *little, size_t big_length) {
+    size_t little_length = strlen(little);
+
+    for (const char *big_end = big + big_length; big < big_end; big++) {
+        if (*big == *little && memcmp(big, little, little_length) == 0) return big;
+    }
+
+    return NULL;
+}
+
+/**
+ * Potentially warn the user if the shebang that has been found to include
+ * "ruby" has a carriage return at the end, as that can cause problems on some
+ * platforms.
+ */
+static void
+pm_parser_warn_shebang_carriage_return(pm_parser_t *parser, const uint8_t *start, size_t length) {
+    if (length > 2 && start[length - 1] == '\n' && start[length - 2] == '\r') {
+        pm_parser_warn(parser, start, start + length, PM_WARN_SHEBANG_CARRIAGE_RETURN);
+    }
+}
+
 /**
  * Initialize a parser with the given start and end pointers.
  */
@@ -19208,22 +19243,6 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm
         // line option
         parser->start_line = options->line;
 
-        // offset option
-        if (options->offset != 0) {
-            const uint8_t *cursor = parser->start;
-            const uint8_t *offset = cursor + options->offset;
-
-            const uint8_t *newline = NULL;
-            while ((newline = next_newline(cursor, parser->end - cursor)) != NULL) {
-                if (newline > offset) break;
-                pm_newline_list_append(&parser->newline_list, newline);
-                cursor = newline + 1;
-            }
-
-            parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = offset, .end = offset };
-            parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = offset, .end = offset };
-        }
-
         // encoding option
         size_t encoding_length = pm_string_length(&options->encoding);
         if (encoding_length > 0) {
@@ -19277,12 +19296,65 @@ pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const pm
         }
     }
 
+    // If the -x command line flag is set, or the first shebang of the file does
+    // not include "ruby", then we'll search for a shebang that does include
+    // "ruby" and start parsing from there.
+    bool search_shebang = PM_PARSER_COMMAND_LINE_OPTION_X(parser);
+
     // If the first two bytes of the source are a shebang, then we'll indicate
     // that the encoding comment is at the end of the shebang.
     if (peek(parser) == '#' && peek_offset(parser, 1) == '!') {
-        const uint8_t *encoding_comment_start = next_newline(source, (ptrdiff_t) size);
-        if (encoding_comment_start) {
-            parser->encoding_comment_start = encoding_comment_start + 1;
+        const uint8_t *newline = next_newline(parser->start, parser->end - parser->start);
+        size_t length = (size_t) ((newline != NULL ? newline : parser->end) - parser->start);
+
+        if (pm_strnstr((const char *) parser->start, "ruby", length) != NULL) {
+            pm_parser_warn_shebang_carriage_return(parser, parser->start, length);
+            if (newline != NULL) parser->encoding_comment_start = newline + 1;
+            search_shebang = false;
+        } else {
+            search_shebang = true;
+        }
+    }
+
+    // Here we're going to find the first shebang that includes "ruby" and start
+    // parsing from there.
+    if (search_shebang) {
+        bool found = false;
+
+        // This is going to point to the start of each line as we check it.
+        // We'll maintain a moving window looking at each line at they come.
+        const uint8_t *cursor = parser->start;
+
+        // The newline pointer points to the end of the current line that we're
+        // considering. If it is NULL, then we're at the end of the file.
+        const uint8_t *newline = next_newline(cursor, parser->end - cursor);
+
+        while (newline != NULL) {
+            pm_newline_list_append(&parser->newline_list, newline);
+
+            cursor = newline + 1;
+            newline = next_newline(cursor, parser->end - cursor);
+
+            size_t length = (size_t) ((newline != NULL ? newline : parser->end) - cursor);
+            if (length > 2 && cursor[0] == '#' && cursor[1] == '!') {
+                if (parser->newline_list.size == 1) {
+                    pm_parser_warn_shebang_carriage_return(parser, cursor, length);
+                }
+
+                if (pm_strnstr((const char *) cursor, "ruby", length) != NULL) {
+                    found = true;
+                    parser->encoding_comment_start = newline + 1;
+                    break;
+                }
+            }
+        }
+
+        if (found) {
+            parser->previous = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor };
+            parser->current = (pm_token_t) { .type = PM_TOKEN_EOF, .start = cursor, .end = cursor };
+        } else {
+            pm_parser_err(parser, parser->start, parser->start, PM_ERR_SCRIPT_NOT_FOUND);
+            pm_newline_list_clear(&parser->newline_list);
         }
     }
 }
diff --git a/src/util/pm_newline_list.c b/src/util/pm_newline_list.c
@@ -19,6 +19,14 @@ pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t capac
     return true;
 }
 
+/**
+ * Clear out the newlines that have been appended to the list.
+ */
+void
+pm_newline_list_clear(pm_newline_list_t *list) {
+    list->size = 1;
+}
+
 /**
  * Append a new offset to the newline list. Returns true if the reallocation of
  * the offsets succeeds (if one was necessary), otherwise returns false.
diff --git a/templates/include/prism/diagnostic.h.erb b/templates/include/prism/diagnostic.h.erb
@@ -66,11 +66,14 @@ typedef struct {
  * The levels of errors generated during parsing.
  */
 typedef enum {
-    /** For errors that cannot be recovered from. */
-    PM_ERROR_LEVEL_FATAL = 0,
+    /** For errors that should raise a syntax error. */
+    PM_ERROR_LEVEL_SYNTAX = 0,
 
     /** For errors that should raise an argument error. */
-    PM_ERROR_LEVEL_ARGUMENT = 1
+    PM_ERROR_LEVEL_ARGUMENT = 1,
+
+    /** For errors that should raise a load error. */
+    PM_ERROR_LEVEL_LOAD = 2
 } pm_error_level_t;
 
 /**
diff --git a/templates/src/diagnostic.c.erb b/templates/src/diagnostic.c.erb
diff --git a/test/prism/command_line_test.rb b/test/prism/command_line_test.rb
diff --git a/test/prism/ruby_api_test.rb b/test/prism/ruby_api_test.rb
