Prepared the lexer to inform file offsets

darcs-hash:20060206184757-49d33-a03145c46d22a9bcfa4e95a249781fff2829815a.gz

net.sf.eclipsefp.haskell.core.jparser.test/src/net/sf/eclipsefp/haskell/core/jparser/test/HaskellPreferenceProviderStub.java

@@ -5,7 +5,7 @@
 public class HaskellPreferenceProviderStub
 	implements IHaskellPreferenceProvider {
 
-	private int fTabSize;
+	private int fTabSize = 8;
 
 	public void setTabSize(int size) {
 		fTabSize = size;

net.sf.eclipsefp.haskell.core.jparser.test/src/net/sf/eclipsefp/haskell/core/jparser/test/LexerTest.java

@@ -8,6 +8,7 @@
 import antlr.Token;
 import antlr.TokenStreamException;
 
+import net.sf.eclipsefp.haskell.core.jparser.EclipseFPToken;
 import net.sf.eclipsefp.haskell.core.jparser.HaskellLexer;
 import net.sf.eclipsefp.haskell.core.jparser.HaskellLexerTokenTypes;
 
@@ -375,6 +376,25 @@ public void testRecognizeVarsymWithAlt() throws TokenStreamException {
 		assertToken(VARIABLE_ID, "var", fLexer.nextToken());
 		assertToken(VARSYM, ".|.", fLexer.nextToken());
 	}
+
+	public void testTokenOffset() throws TokenStreamException {
+		final String input = "module Main where\n" +
+                             "\tfat 0 = 1";
+		fLexer = createLexer(input);
+
+		assertOffset(0, fLexer.nextToken());
+		assertOffset(7, fLexer.nextToken());
+
+		// where \n
+		fLexer.skipTokens(2);
+
+		assertOffset(19, fLexer.nextToken());
+	}
+
+	private void assertOffset(int expectedOffset, Token token) {
+		EclipseFPToken tok = (EclipseFPToken) token;
+		assertEquals(expectedOffset, tok.getOffset());
+	}
 
 	// TODO escape  -> 	 \ ( charesc | ascii | decimal | o octal | x hexadecimal )
 	//      charesc -> 	a | b | f | n | r | t | v | \ | " | ' | &

net.sf.eclipsefp.haskell.core.jparser/antlr-src/haskell-lexer.g

@@ -56,12 +56,15 @@ tokens {
 	{
 		try {
 			this.setTabSize(new HaskellPreferenceProvider().getTabSize());
-		} catch(NullPointerException e) {
+		} catch(Exception e) {
 			//happens when the eclipse platform isn't loaded
 			//in this case, ignore the default preference provider
 		}
 	}
 
+	private long tokenStartOffset;
+	private long currentOffset;
+	
 	public HaskellLexer(Reader reader, IHaskellPreferenceProvider prefs) {
 		this(reader);
 		this.setTabSize(prefs.getTabSize());
@@ -70,12 +73,23 @@ tokens {
 	/* workaround for starting token coordinates from 0
 	 * as eclipse expects them to */
     protected Token makeToken(int t) {
-    	Token result = super.makeToken(t);
-    	
-    	result.setLine(result.getLine() - 1);
-    	result.setColumn(result.getColumn() - 1);
-    	
-    	return result;
+        EclipseFPToken tok = new EclipseFPToken();
+        tok.setType(t);
+        tok.setColumn(getInputState().getTokenStartColumn() - 1);
+        tok.setLine(getInputState().getTokenStartLine() - 1);
+        tok.setOffset(tokenStartOffset);
+        
+        return tok;
+    }
+    
+    public void consume() throws CharStreamException {
+        super.consume();
+        currentOffset++;
+    }
+    
+    public void resetText() {
+        super.resetText();
+        tokenStartOffset = currentOffset;
     }
 }
 

net.sf.eclipsefp.haskell.core.jparser/src/net/sf/eclipsefp/haskell/core/jparser/EclipseFPToken.java

@@ -0,0 +1,17 @@
+package net.sf.eclipsefp.haskell.core.jparser;
+
+import antlr.CommonToken;
+
+public class EclipseFPToken extends CommonToken {
+
+	private long fOffset;
+
+	public long getOffset() {
+		return fOffset;
+	}
+
+	public void setOffset(long offset) {
+		fOffset = offset;
+	}
+
+}