Implement the offset part of detailed source sections.

It's not always correct, but it only leads to four failing Truffle specs
for keywords that the parser then asks the lexer again for a position,
such as __FILE__.

core/src/main/java/org/jruby/lexer/yacc/InputStreamLexerSource.java

@@ -11,8 +11,6 @@
 public class InputStreamLexerSource extends LexerSource {
     private static final int INITIAL_PUSHBACK_SIZE = 100;
 
-    public static final int DATA_READ_BUFFER_SIZE = 65536;
-
     // Where we get our newest char's
     private final InputStream in;
 
@@ -102,7 +100,6 @@ public boolean peek(int to) throws IOException {
     }
 
     private void advance(int c) {
-
         twoAgo = oneAgo;
         oneAgo = c;
         offset++;
@@ -133,7 +130,6 @@ private void growBuf() {
     }
 
     private void retreat() {
-
         offset--;
         oneAgo = twoAgo;
         twoAgo = 0;

core/src/main/java/org/jruby/lexer/yacc/LexerSource.java

@@ -38,6 +38,7 @@
 
 import org.jruby.parser.ParserConfiguration;
 import org.jruby.util.ByteList;
+import org.jruby.util.cli.Options;
 
 /**
  * This class is what feeds the lexer.  It is primarily a wrapper around a
@@ -49,9 +50,6 @@
  * 
  */
 public abstract class LexerSource {
-
-    // Where we get new positions from.
-	private SourcePositionFactory positionFactory;
 
     // The name of this source (e.g. a filename: foo.rb)
     private final String sourceName;
@@ -74,6 +72,11 @@ public abstract class LexerSource {
     // Last full line read.
     private StringBuilder sourceLine;
 
+    protected ISourcePosition lastPosition;
+    private int startOfTokenOffset;
+
+    private boolean detailedSourcePositions = Options.PARSER_DETAILED_SOURCE_POSITIONS.load();
+
     /**
      * Create our food-source for the lexer
      * 
@@ -82,10 +85,15 @@ public abstract class LexerSource {
     protected LexerSource(String sourceName, List<String> list, int lineOffset) {
         this.sourceName = sourceName;
         this.lineOffset = lineOffset;
-        positionFactory = new SimpleSourcePositionFactory(this, line);
         this.list = list;
         lineBuffer = new StringBuilder(160);
         sourceLine = new StringBuilder(160);
+
+        if (detailedSourcePositions) {
+            lastPosition = new DetailedSourcePosition(sourceName, line, 0, 0);
+        } else {
+            lastPosition = new SimpleSourcePosition(sourceName, line);
+        }
     }
 
     /**
@@ -117,6 +125,12 @@ public int getOffset() {
         return (offset <= 0 ? 0 : offset);
     }
 
+    public void startOfToken() {
+        if (detailedSourcePositions) {
+            startOfTokenOffset = offset;
+        }
+    }
+
     /**
      * Where is the reader within the source {filename,row}
      *
@@ -125,8 +139,30 @@ public int getOffset() {
      * @return the current position
      */
     public ISourcePosition getPosition(ISourcePosition startPosition) {
-        ISourcePosition sourcePosition = positionFactory.getPosition(startPosition);
-        return sourcePosition;
+        if (detailedSourcePositions) {
+            if (startPosition == null) {
+                lastPosition = new DetailedSourcePosition(getFilename(), getVirtualLine(), startOfTokenOffset, offset - startOfTokenOffset);
+            } else {
+                DetailedSourcePosition detailedStartPosition = (DetailedSourcePosition) startPosition;
+                lastPosition = new DetailedSourcePosition(getFilename(), getVirtualLine(), detailedStartPosition.getOffset(), 0); // offset - detailedStartPosition.getOffset()
+                return lastPosition;
+            }
+        } else {
+            if (startPosition == null) {
+                // Only give new position if we are at least one char past \n of previous line so that last tokens
+                // of previous line will not get associated with the next line.
+                if (lastPosition.getLine() == getVirtualLine() || lastWasBeginOfLine()) {
+                    return lastPosition;
+                }
+
+                lastPosition = new SimpleSourcePosition(getFilename(), getVirtualLine());
+            } else {
+                lastPosition = startPosition;
+                return lastPosition;
+            }
+        }
+
+        return lastPosition;
     }
 
     /**
@@ -267,4 +303,5 @@ public int readCodepoint(int first, Encoding encoding) throws IOException {
     public abstract boolean lastWasBeginOfLine();
     public abstract boolean wasBeginOfLine();
     public abstract InputStream getRemainingAsStream() throws IOException;
+
 }

core/src/main/java/org/jruby/lexer/yacc/RubyLexer.java

@@ -308,9 +308,14 @@ public static Keyword getKeyword(String str) {
     private LexState last_state;
     public ISourcePosition tokline;
 
+    public void startOfToken() {
+        src.startOfToken();
+    }
+
     public void newtok() {
         tokline = getPosition();
     }
+
     // Tempory buffer to build up a potential token.  Consumer takes responsibility to reset 
     // this before use.
     private StringBuilder tokenBuffer = new StringBuilder(60);
@@ -1084,6 +1089,7 @@ private int yylex() throws IOException {
         commandStart = false;
 
         loop: for(;;) {
+            startOfToken();
             last_state = lex_state;
             c = src.read();
             switch(c) {

core/src/main/java/org/jruby/lexer/yacc/SimpleSourcePosition.java

@@ -29,6 +29,7 @@
 package org.jruby.lexer.yacc;
 
 public class SimpleSourcePosition implements ISourcePosition {
+
     final String filename;
     final int line;
 

core/src/main/java/org/jruby/truffle/translator/Translator.java

@@ -11,6 +11,7 @@
 
 import com.oracle.truffle.api.source.Source;
 import com.oracle.truffle.api.source.SourceSection;
+import org.jruby.lexer.yacc.DetailedSourcePosition;
 import org.jruby.lexer.yacc.InvalidSourcePosition;
 import org.jruby.truffle.nodes.RubyNode;
 import org.jruby.truffle.runtime.RubyContext;
@@ -41,6 +42,9 @@ public SourceSection translate(Source source, org.jruby.lexer.yacc.ISourcePositi
             } else {
                 return parentSourceSection;
             }
+        } else if (sourcePosition instanceof DetailedSourcePosition) {
+            final DetailedSourcePosition detailedSourcePosition = (DetailedSourcePosition) sourcePosition;
+            return source.createSection(getIdentifier(), detailedSourcePosition.getOffset(), detailedSourcePosition.getLength());
         } else if (Options.TRUFFLE_ALLOW_SIMPLE_SOURCE_SECTIONS.load()) {
             // If we didn't run with -X+T, so maybe we're using truffelize, we might still get simple source sections
             return source.createSection(getIdentifier(), sourcePosition.getLine() + 1);

core/src/main/java/org/jruby/util/cli/ArgumentProcessor.java

@@ -373,6 +373,7 @@ private void processArgument() {
                         checkGraalVersion();
                         config.setCompileMode(RubyInstanceConfig.CompileMode.TRUFFLE);
                         config.setDisableGems(true);
+                        Options.PARSER_DETAILED_SOURCE_POSITIONS.force(Boolean.toString(true));
                     } else if (extendedOption.endsWith("...")) {
                         Options.listPrefix(extendedOption.substring(0, extendedOption.length() - "...".length()));
                         config.setShouldRunInterpreter(false);

core/src/main/java/org/jruby/util/cli/Options.java

@@ -58,13 +58,13 @@ public class Options {
     // This section holds all Options for JRuby. They will be listed in the
     // --properties output.
 
+    public static final Option<Boolean> PARSER_DETAILED_SOURCE_POSITIONS = bool(PARSER, "parser.detailed_source_positions", false, "Produce detailed source positions");
     public static final Option<Boolean> PARSER_WARN_USELESSS_USE_OF = bool(PARSER, "parser.warn.useless_use_of", true, "Warn about potentially useless expressions in void contents.");
     public static final Option<Boolean> PARSER_WARN_NOT_REACHED = bool(PARSER, "parser.warn.not_reached", true, "Warn about statements that can never be reached.");
     public static final Option<Boolean> PARSER_WARN_GROUPED_EXPRESSIONS = bool(PARSER, "parser.warn.grouped_expressions", true, "Warn about interpreting (...) as a grouped expression.");
     public static final Option<Boolean> PARSER_WARN_LOCAL_SHADOWING = bool(PARSER, "parser.warn.shadowing_local", true, "Warn about shadowing local variables.");
     public static final Option<Boolean> PARSER_WARN_REGEX_CONDITION = bool(PARSER, "parser.warn.regex_condition", true, "Warn about regex literals in conditions.");
     public static final Option<Boolean> PARSER_WARN_ARGUMENT_PREFIX = bool(PARSER, "parser.warn.argument_prefix", true, "Warn about splat operators being interpreted as argument prefixes.");
-    public static final Option<Boolean> PARSER_ALWAYS_TRUFFLE_POSITIONS = bool(PARSER, "parser.always_truffle_positions", false, "Always generate Truffle source positions, even if we're not -X+T.");
 
     public static final Option<CompileMode> COMPILE_MODE = enumeration(COMPILER, "compile.mode", CompileMode.class, CompileMode.JIT, "Set compilation mode. JIT = at runtime; FORCE = before execution.");
     public static final Option<Boolean> COMPILE_DUMP = bool(COMPILER, "compile.dump", false, "Dump to console all bytecode generated at runtime.");
@@ -139,7 +139,7 @@ public class Options {
     public static final Option<Integer> TRUFFLE_ARRAYS_SMALL = integer(TRUFFLE, "truffle.arrays.small", 3, "Maximum size of an Array to consider small for optimisations.");
     public static final Option<Integer> TRUFFLE_HASHES_SMALL = integer(TRUFFLE, "truffle.hashes.small", 3, "Maximum size of a Hash to consider small for optimisations.");
     public static final Option<Boolean> TRUFFLE_COMPILER_PASS_LOOPS_THROUGH_BLOCKS = bool(TRUFFLE, "truffle.compiler.pass_loops_through_blocks", false, "Pass loop counts through blocks to the method that is calling the block.");
-    public static final Option<Boolean> TRUFFLE_ALLOW_SIMPLE_SOURCE_SECTIONS = bool(TRUFFLE, "truffle.allow_simple_source_sections", true, "Allow simple source sections.");
+    public static final Option<Boolean> TRUFFLE_ALLOW_SIMPLE_SOURCE_SECTIONS = bool(TRUFFLE, "truffle.allow_simple_source_sections", false, "Allow simple source sections.");
     public static final Option<TruffleBridge.BacktraceFormatter> TRUFFLE_BACKTRACE_DISPLAY_FORMAT = enumeration(TRUFFLE, "truffle.backtrace.display_format", TruffleBridge.BacktraceFormatter.class, TruffleBridge.BacktraceFormatter.MRI, "How to format backtraces displayed to the user.");
     public static final Option<TruffleBridge.BacktraceFormatter> TRUFFLE_BACKTRACE_DEBUG_FORMAT = enumeration(TRUFFLE, "truffle.backtrace.debug_format", TruffleBridge.BacktraceFormatter.class, TruffleBridge.BacktraceFormatter.DEBUG, "How to format backtraces displayed using TruffleDebug.dump_call_stack.");
     public static final Option<TruffleBridge.BacktraceFormatter> TRUFFLE_BACKTRACE_EXCEPTION_FORMAT = enumeration(TRUFFLE, "truffle.backtrace.exception_format", TruffleBridge.BacktraceFormatter.class, TruffleBridge.BacktraceFormatter.MRI, "How to format backtraces in Exception objects.");