Repurpose jit.maxSize to add a size limit for jitting methods.

#3016 was caused by too-large methods entering the JIT
pipeline and consuming too much memory, mostly within ASM's data
structures. In this case one particularly large method was the
Ragel-generated "advance" method in @whitequark's parser library,
which amounts to over 17k IR instructions.

The default instruction count max of 2000 here is based on running
'gem install rails', 'rails new ...' and 'rake test:mri:jit' and
monitoring methods which exceed the limit; during this process,
all but a handful of methods are under 2000 instructions. We
believe this limit will allow important methods to still JIT while
avoiding extremely large outliers causing JIT pipeline or memory
pressure.

Fixes #3016.

core/src/main/java/org/jruby/compiler/JITCompiler.java

@@ -38,6 +38,7 @@
 import org.jruby.internal.runtime.methods.CompiledIRMethod;
 import org.jruby.internal.runtime.methods.MixedModeIRMethod;
 import org.jruby.ir.IRMethod;
+import org.jruby.ir.interpreter.InterpreterContext;
 import org.jruby.ir.targets.JVMVisitor;
 import org.jruby.runtime.ThreadContext;
 import org.jruby.runtime.builtin.IRubyObject;
@@ -363,7 +364,13 @@ protected void compile() {
             // Time the compilation
             long start = System.nanoTime();
 
-            method.ensureInstrsReady();
+            InterpreterContext ic = method.ensureInstrsReady();
+
+            int insnCount = ic.getInstructions().length;
+            if (insnCount > Options.JIT_MAXSIZE.load()) {
+                // methods with more than our limit of basic blocks are likely too large to JIT, so bail out
+                throw new NotCompilableException("Could not compile " + method + "; instruction count " + insnCount + " exceeds threshold of " + Options.JIT_MAXSIZE.load());
+            }
 
             // This may not be ok since we'll end up running passes specific to JIT
             // CON FIXME: Really should clone scope before passes in any case

core/src/main/java/org/jruby/util/cli/Options.java

@@ -104,7 +104,7 @@ public class Options {
 
     public static final Option<Integer> JIT_THRESHOLD = integer(JIT, "jit.threshold", Constants.JIT_THRESHOLD, "Set the JIT threshold to the specified method invocation count.");
     public static final Option<Integer> JIT_MAX = integer(JIT, "jit.max", Constants.JIT_MAX_METHODS_LIMIT, "Set the max count of active methods eligible for JIT-compilation.");
-    public static final Option<Integer> JIT_MAXSIZE = integer(JIT, "jit.maxsize", Constants.JIT_MAX_SIZE_LIMIT, "Set the maximum full-class byte size allowed for jitted methods.");
+    public static final Option<Integer> JIT_MAXSIZE = integer(JIT, "jit.maxsize", Constants.JIT_MAX_SIZE_LIMIT, "Set the max size (in IR instructions) for a method to be eligible to JIT.");
     public static final Option<Boolean> JIT_LOGGING = bool(JIT, "jit.logging", false, "Enable JIT logging (reports successful compilation).");
     public static final Option<Boolean> JIT_LOGGING_VERBOSE = bool(JIT, "jit.logging.verbose", false, "Enable verbose JIT logging (reports failed compilation).");
     public static final Option<Boolean> JIT_DUMPING = bool(JIT, "jit.dumping", false, "Enable stdout dumping of JITed bytecode.");