Allow to mixed and specify *.rb and *.mrb in bin/mruby

It is not decides by the extension.
In order to be recognized as a `.mrb` file, the following three points must be satisfied:
- File starts with "RITE"
- At least `sizeof(struct rite_binary_header)` bytes can be read
- `NUL` is included in the first 64 bytes of the file
If these are not met, it is judged as a text file and it is processed as a Ruby script.

The `bin/mruby -b` switch is still available which treats the given file as a `.mrb` file.

New `MRB_API` function:
- `include/mruby/compile.h` and `mrbgems/mruby-compiler/core/parse.y`
  - `mrb_load_detect_file_cxt()` (remove with `MRB_DISABLE_STDIO`)

NOTE:
- Even script files now always open in binary mode for `bin/mruby`.
  The `\r\n` is handled by the `nextc()` function already, so there is no problem even on Windows.
- The `nextc0()` function in `mrbgems/mruby-compiler/core/parse.y` can now specify a string buffer and a file pointer at the same time.
  In this case, get it from the string buffer first.

This patch includes modifies by comment of #5157.

include/mruby/compile.h

@@ -116,6 +116,7 @@ struct mrb_parser_state {
   mrb_ast_node *cells;
   const char *s, *send;
 #ifndef MRB_DISABLE_STDIO
+  /* If both f and s are non-null, it will be taken preferentially from s until s < send. */
   FILE *f;
 #endif
   mrbc_context *cxt;
@@ -193,6 +194,7 @@ MRB_API mrb_value mrb_load_exec(mrb_state *mrb, struct mrb_parser_state *p, mrbc
 #ifndef MRB_DISABLE_STDIO
 MRB_API mrb_value mrb_load_file(mrb_state*,FILE*);
 MRB_API mrb_value mrb_load_file_cxt(mrb_state*,FILE*, mrbc_context *cxt);
+MRB_API mrb_value mrb_load_detect_file_cxt(mrb_state *mrb, FILE *fp, mrbc_context *c);
 #endif
 MRB_API mrb_value mrb_load_string(mrb_state *mrb, const char *s);
 MRB_API mrb_value mrb_load_nstring(mrb_state *mrb, const char *s, size_t len);

mrbgems/mruby-bin-mruby/bintest/mruby.rb

@@ -19,7 +19,7 @@ def assert_mruby(exp_out, exp_err, exp_success, args)
   script, bin = Tempfile.new('test.rb'), Tempfile.new('test.mrb')
   File.write script.path, 'p "ok"'
   system "#{cmd('mrbc')} -g -o #{bin.path} #{script.path}"
-  o = `#{cmd('mruby')} -b #{bin.path}`.strip
+  o = `#{cmd('mruby')} #{bin.path}`.strip
   assert_equal '"ok"', o
 end
 
@@ -33,7 +33,7 @@ def assert_mruby(exp_out, exp_err, exp_success, args)
 
   # .mrb file
   `#{cmd('mrbc')} -o "#{bin.path}" "#{script.path}"`
-  assert_equal "\"#{bin.path}\"", `#{cmd('mruby')} -b "#{bin.path}"`.chomp
+  assert_equal "\"#{bin.path}\"", `#{cmd('mruby')} "#{bin.path}"`.chomp
 
   # one liner
   assert_equal '"-e"', `#{cmd('mruby')} -e #{shellquote('p $0')}`.chomp
@@ -48,7 +48,7 @@ def assert_mruby(exp_out, exp_err, exp_success, args)
   script, bin = Tempfile.new('test.rb'), Tempfile.new('test.mrb')
   File.write script.path, 'p [3.21, 2e308.infinite?, -2e308.infinite?]'
   system "#{cmd('mrbc')} -g -o #{bin.path} #{script.path}"
-  assert_equal "[3.21, 1, -1]", `#{cmd('mruby')} -b #{bin.path}`.chomp!
+  assert_equal "[3.21, 1, -1]", `#{cmd('mruby')} #{bin.path}`.chomp!
 end
 
 assert '__END__', '8.6' do

mrbgems/mruby-bin-mruby/tools/mruby/mruby.c

@@ -12,6 +12,11 @@
 #include <mruby/variable.h>
 #include <mruby/proc.h>
 
+#if defined(_WIN32) || defined(_WIN64)
+# include <io.h> /* for setmode */
+# include <fcntl.h>
+#endif
+
 struct _args {
   FILE *rfp;
   char *cmdline;
@@ -218,7 +223,7 @@ parse_args(mrb_state *mrb, int argc, char **argv, struct _args *args)
     }
     else {
       args->rfp = strcmp(argv[0], "-") == 0 ?
-        stdin : fopen(argv[0], args->mrbfile ? "rb" : "r");
+        stdin : fopen(argv[0], "rb");
       if (args->rfp == NULL) {
         fprintf(stderr, "%s: Cannot open program file: %s\n", opts->program, argv[0]);
         return EXIT_FAILURE;
@@ -228,6 +233,11 @@ parse_args(mrb_state *mrb, int argc, char **argv, struct _args *args)
       argc--; argv++;
     }
   }
+#if defined(_WIN32) || defined(_WIN64)
+  if (args->rfp == stdin) {
+    setmode(_fileno(stdin), O_BINARY);
+  }
+#endif
   args->argv = (char **)mrb_realloc(mrb, args->argv, sizeof(char*) * (argc + 1));
   memcpy(args->argv, argv, (argc+1) * sizeof(char*));
   args->argc = argc;
@@ -309,7 +319,7 @@ main(int argc, char **argv)
     /* Load libraries */
     for (i = 0; i < args.libc; i++) {
       struct REnv *e;
-      FILE *lfp = fopen(args.libv[i], args.mrbfile ? "rb" : "r");
+      FILE *lfp = fopen(args.libv[i], "rb");
       if (lfp == NULL) {
         fprintf(stderr, "%s: Cannot open library file: %s\n", *argv, args.libv[i]);
         mrbc_context_free(mrb, c);
@@ -320,7 +330,7 @@ main(int argc, char **argv)
         v = mrb_load_irep_file_cxt(mrb, lfp, c);
       }
       else {
-        v = mrb_load_file_cxt(mrb, lfp, c);
+        v = mrb_load_detect_file_cxt(mrb, lfp, c);
       }
       fclose(lfp);
       e = mrb->c->cibase->env;
@@ -334,7 +344,7 @@ main(int argc, char **argv)
       v = mrb_load_irep_file_cxt(mrb, args.rfp, c);
     }
     else if (args.rfp) {
-      v = mrb_load_file_cxt(mrb, args.rfp, c);
+      v = mrb_load_detect_file_cxt(mrb, args.rfp, c);
     }
     else {
       char* utf8 = mrb_utf8_from_locale(args.cmdline, -1);

mrbgems/mruby-bin-strip/bintest/mruby-strip.rb

@@ -32,7 +32,7 @@
   o = `#{cmd('mruby-strip')} #{compiled1.path}`
   assert_equal 0, $?.exitstatus
   assert_equal "", o
-  assert_equal `#{cmd('mruby')} #{script_file.path}`, `#{cmd('mruby')} -b #{compiled1.path}`
+  assert_equal `#{cmd('mruby')} #{script_file.path}`, `#{cmd('mruby')} #{compiled1.path}`
 
   o = `#{cmd('mruby-strip')} #{compiled1.path} #{compiled2.path}`
   assert_equal 0, $?.exitstatus

mrbgems/mruby-compiler/core/parse.y

@@ -21,6 +21,7 @@
 #include <mruby/error.h>
 #include <mruby/throw.h>
 #include <mruby/string.h>
+#include <mruby/dump.h>
 #include "node.h"
 
 #define YYLEX_PARAM p
@@ -4122,20 +4123,20 @@ static inline int
 nextc0(parser_state *p)
 {
   int c;
-#ifndef MRB_DISABLE_STDIO
-  if (p->f) {
-    if (feof(p->f)) return -1;
-    c = fgetc(p->f);
-    if (c == EOF) return -1;
+
+  if (p->s && p->s < p->send) {
+    c = (unsigned char)*p->s++;
   }
-  else
+  else {
+#ifndef MRB_DISABLE_STDIO
+    if (p->f) {
+      c = fgetc(p->f);
+      if (feof(p->f)) return -1;
+    }
+    else
 #endif
-    if (!p->s || p->s >= p->send) {
       return -1;
-    }
-    else {
-      c = (unsigned char)*p->s++;
-    }
+  }
   return c;
 }
 
@@ -6520,19 +6521,26 @@ mrb_parser_get_filename(struct mrb_parser_state* p, uint16_t idx) {
 }
 
 #ifndef MRB_DISABLE_STDIO
-MRB_API parser_state*
-mrb_parse_file(mrb_state *mrb, FILE *f, mrbc_context *c)
+static struct mrb_parser_state *
+mrb_parse_file_continue(mrb_state *mrb, FILE *f, const void *prebuf, size_t prebufsize, mrbc_context *c)
 {
   parser_state *p;
 
   p = mrb_parser_new(mrb);
   if (!p) return NULL;
-  p->s = p->send = NULL;
+  p->s = (const char *)prebuf;
+  p->send = (const char *)prebuf + prebufsize;
   p->f = f;
 
   mrb_parser_parse(p, c);
   return p;
 }
+
+MRB_API parser_state*
+mrb_parse_file(mrb_state *mrb, FILE *f, mrbc_context *c)
+{
+  return mrb_parse_file_continue(mrb, f, NULL, 0, c);
+}
 #endif
 
 MRB_API parser_state*
@@ -6629,6 +6637,55 @@ mrb_load_file(mrb_state *mrb, FILE *f)
 {
   return mrb_load_file_cxt(mrb, f, NULL);
 }
+
+#define DETECT_SIZE 64
+
+/*
+ * In order to be recognized as a `.mrb` file, the following three points must be satisfied:
+ * - File starts with "RITE"
+ * - At least `sizeof(struct rite_binary_header)` bytes can be read
+ * - `NUL` is included in the first 64 bytes of the file
+ */
+MRB_API mrb_value
+mrb_load_detect_file_cxt(mrb_state *mrb, FILE *fp, mrbc_context *c)
+{
+  union {
+    char b[DETECT_SIZE];
+    struct rite_binary_header h;
+  } leading;
+  size_t bufsize;
+
+  if (mrb == NULL || fp == NULL) {
+    return mrb_nil_value();
+  }
+
+  bufsize = fread(leading.b, sizeof(char), sizeof(leading), fp);
+  if (bufsize < sizeof(leading.h) ||
+      memcmp(leading.h.binary_ident, RITE_BINARY_IDENT, sizeof(leading.h.binary_ident)) != 0 ||
+      memchr(leading.b, '\0', bufsize) == NULL) {
+    return mrb_load_exec(mrb, mrb_parse_file_continue(mrb, fp, leading.b, bufsize, c), c);
+  }
+  else {
+    size_t binsize;
+    uint8_t *bin;
+    mrb_value bin_obj = mrb_nil_value(); /* temporary string object */
+    mrb_value result;
+
+    binsize = bin_to_uint32(leading.h.binary_size);
+    bin_obj = mrb_str_new(mrb, NULL, binsize);
+    bin = (uint8_t *)RSTRING_PTR(bin_obj);
+    memcpy(bin, leading.b, bufsize);
+    if (binsize > bufsize &&
+        fread(bin + bufsize, binsize - bufsize, 1, fp) == 0) {
+      binsize = bufsize;
+      /* The error is reported by mrb_load_irep_buf_cxt() */
+    }
+
+    result = mrb_load_irep_buf_cxt(mrb, bin, binsize, c);
+    if (mrb_string_p(bin_obj)) mrb_str_resize(mrb, bin_obj, 0);
+    return result;
+  }
+}
 #endif
 
 MRB_API mrb_value