Added character position to token data structure.

mysteriouspants · Mar 4, 2012 · 287b681 · 287b681
1 parent 32dc81b
commit 287b681
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 4 deletions.
diff --git a/CoreParse/Tokenisation/CPTokeniser.m b/CoreParse/Tokenisation/CPTokeniser.m
@@ -119,6 +119,7 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
             {
                 [tok setLineNumber:currentLineNumber];
                 [tok setColumnNumber:currentColumnNumber];
+                [tok setCharacterNumber:lastTokenOffset];
 
                 if ([delegate respondsToSelector:@selector(tokeniser:shouldConsumeToken:)])
                 {
@@ -169,6 +170,7 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
         CPEOFToken *token = [CPEOFToken eof];
         [token setLineNumber:currentLineNumber];
         [token setColumnNumber:currentColumnNumber];
+        [token setCharacterNumber:inputLength];
         [stream pushToken:token];
     }
     [stream closeTokenStream];

diff --git a/CoreParse/Tokenisation/Token Types/CPToken.h b/CoreParse/Tokenisation/Token Types/CPToken.h
@@ -32,4 +32,9 @@
  */
 @property (readwrite, assign) NSUInteger columnNumber;
 
+/**
+ * The index in the input string of the first character in this token.
+ */
+@property (readwrite, assign) NSUInteger characterNumber;
+
 @end
diff --git a/CoreParse/Tokenisation/Token Types/CPToken.m b/CoreParse/Tokenisation/Token Types/CPToken.m
@@ -12,6 +12,7 @@ @implementation CPToken
 
 @synthesize lineNumber;
 @synthesize columnNumber;
+@synthesize characterNumber;
 
 - (NSString *)name
 {

diff --git a/CoreParseTests/CoreParseTests.m b/CoreParseTests/CoreParseTests.m
@@ -301,14 +301,16 @@ - (void)testTokenLineColumnNumbers
     [tokeniser addTokenRecogniser:[CPIdentifierRecogniser identifierRecogniser]];
     [tokeniser addTokenRecogniser:[CPWhiteSpaceRecogniser whiteSpaceRecogniser]];
     CPTokenStream *tokenStream = [tokeniser tokenise:@"/* blah\nblah blah\n blah */ long jam\n\nlong ham"];
-    NSUInteger tokenLines[]   = {0, 2, 2, 2 , 2 , 2 , 4, 4, 4, 4};
-    NSUInteger tokenColumns[] = {0, 8, 9, 13, 14, 17, 0, 4, 5, 8};
+    NSUInteger tokenLines[]     = {0, 2 , 2 , 2 , 2 , 2 , 4 , 4 , 4 , 4 };
+    NSUInteger tokenColumns[]   = {0, 8 , 9 , 13, 14, 17, 0 , 4 , 5 , 8 };
+    NSUInteger tokenPositions[] = {0, 26, 27, 31, 32, 35, 37, 41, 42, 45};
     NSUInteger tokenNumber = 0;
     CPToken *token = nil;
     while ((token = [tokenStream popToken]))
     {
-        STAssertEquals([token lineNumber  ], tokenLines  [tokenNumber], @"Line number for token %lu is incorrect", tokenNumber, nil);
-        STAssertEquals([token columnNumber], tokenColumns[tokenNumber], @"Column number for toen %lu is incorrect", tokenNumber, nil);
+        STAssertEquals([token lineNumber     ], tokenLines  [tokenNumber]  , @"Line number for token %lu is incorrect", tokenNumber, nil);
+        STAssertEquals([token columnNumber   ], tokenColumns[tokenNumber]  , @"Column number for token %lu is incorrect", tokenNumber, nil);
+        STAssertEquals([token characterNumber], tokenPositions[tokenNumber], @"Character nmber for token %lu is incorrect", tokenNumber, nil);
         tokenNumber++;
     }
 }