Added line and column numbers, fixes #12

beelsebob · Feb 5, 2012 · 45368c8 · 45368c8
1 parent 7c941b8
commit 45368c8
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 1 deletion.
diff --git a/CoreParse/Tokenisation/CPTokeniser.m b/CoreParse/Tokenisation/CPTokeniser.m
@@ -19,6 +19,7 @@ @interface CPTokeniser ()
 @property (readwrite, retain) NSMutableArray *tokenRecognisers;
 
 - (void)addToken:(CPToken *)tok toStream:(CPTokenStream *)stream;
+- (void)advanceLineNumber:(NSUInteger *)ln columnNumber:(NSUInteger *)cn withInput:(NSString *)input range:(NSRange)range;
 
 @end
 
@@ -102,6 +103,8 @@ - (CPTokenStream *)tokenise:(NSString *)input
 - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
 {
     NSUInteger currentTokenOffset = 0;
+    NSUInteger currentLineNumber = 0;
+    NSUInteger currentColumnNumber = 0;
     NSUInteger inputLength = [input length];
     NSArray *recs = [self tokenRecognisers];
 
@@ -114,11 +117,15 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
             CPToken *tok = [recogniser recogniseTokenInString:input currentTokenPosition:&currentTokenOffset];
             if (nil != tok)
             {
+                [tok setLineNumber:currentLineNumber];
+                [tok setColumnNumber:currentColumnNumber];
+
                 if ([delegate respondsToSelector:@selector(tokeniser:shouldConsumeToken:)])
                 {
                     if ([delegate tokeniser:self shouldConsumeToken:tok])
                     {
                         [self addToken:tok toStream:stream];
+                        [self advanceLineNumber:&currentLineNumber columnNumber:&currentColumnNumber withInput:input range:NSMakeRange(lastTokenOffset, currentTokenOffset - lastTokenOffset)];
                         recognised = YES;
                         break;
                     }
@@ -130,6 +137,7 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
                 else
                 {
                     [self addToken:tok toStream:stream];
+                    [self advanceLineNumber:&currentLineNumber columnNumber:&currentColumnNumber withInput:input range:NSMakeRange(lastTokenOffset, currentTokenOffset - lastTokenOffset)];
                     recognised = YES;
                     break;
                 }
@@ -158,7 +166,10 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
     }
     if (inputLength <= currentTokenOffset)
     {
-        [stream pushToken:[CPEOFToken eof]];
+        CPEOFToken *token = [CPEOFToken eof];
+        [token setLineNumber:currentLineNumber];
+        [token setColumnNumber:currentColumnNumber];
+        [stream pushToken:token];
     }
     [stream closeTokenStream];
 }
@@ -177,4 +188,27 @@ - (void)addToken:(CPToken *)tok toStream:(CPTokenStream *)stream
     [stream pushTokens:toks];
 }
 
+- (void)advanceLineNumber:(NSUInteger *)ln columnNumber:(NSUInteger *)cn withInput:(NSString *)input range:(NSRange)range
+{
+    NSRange searchRange = range;
+    NSUInteger rangeEnd = range.location + range.length;
+    NSRange foundRange = [input rangeOfCharacterFromSet:[NSCharacterSet characterSetWithCharactersInString:@"\n\r"] options:NSLiteralSearch range:searchRange];
+    NSUInteger lastNewLineLocation = NSNotFound;
+    while (foundRange.location != NSNotFound)
+    {
+        *ln += foundRange.length;
+        lastNewLineLocation = foundRange.location + foundRange.length;
+        searchRange = NSMakeRange(lastNewLineLocation, rangeEnd - lastNewLineLocation);
+        foundRange = [input rangeOfCharacterFromSet:[NSCharacterSet characterSetWithCharactersInString:@"\n\r"] options:NSLiteralSearch range:searchRange];
+    }
+    if (lastNewLineLocation != NSNotFound)
+    {
+        *cn = rangeEnd - lastNewLineLocation;
+    }
+    else
+    {
+        *cn += range.length;
+    }
+}
+
 @end
diff --git a/CoreParse/Tokenisation/Token Types/CPToken.h b/CoreParse/Tokenisation/Token Types/CPToken.h
@@ -22,4 +22,14 @@
  */
 @property (readonly) NSString *name;
 
+/**
+ * The line on which the token can be found.
+ */
+@property (readwrite, assign) NSUInteger lineNumber;
+
+/**
+ * The column on which the token can be found.
+ */
+@property (readwrite, assign) NSUInteger columnNumber;
+
 @end
diff --git a/CoreParse/Tokenisation/Token Types/CPToken.m b/CoreParse/Tokenisation/Token Types/CPToken.m
@@ -10,6 +10,9 @@
 
 @implementation CPToken
 
+@synthesize lineNumber;
+@synthesize columnNumber;
+
 - (NSString *)name
 {
     [NSException raise:@"Abstract method called exception" format:@"CPToken is abstract, and should not have name called."];

diff --git a/CoreParseTests/CoreParseTests.m b/CoreParseTests/CoreParseTests.m
@@ -293,6 +293,26 @@ - (void)testTokeniserError
     STAssertEqualObjects(tokenStream, expectedTokenStream, @"Inserting error token and continuing according to delegate failed.", nil);
 }
 
+- (void)testTokenLineColumnNumbers
+{
+    CPTokeniser *tokeniser = [[[CPTokeniser alloc] init] autorelease];
+    [tokeniser addTokenRecogniser:[CPQuotedRecogniser quotedRecogniserWithStartQuote:@"/*" endQuote:@"*/" name:@"Comment"]];
+    [tokeniser addTokenRecogniser:[CPKeywordRecogniser recogniserForKeyword:@"long"]];
+    [tokeniser addTokenRecogniser:[CPIdentifierRecogniser identifierRecogniser]];
+    [tokeniser addTokenRecogniser:[CPWhiteSpaceRecogniser whiteSpaceRecogniser]];
+    CPTokenStream *tokenStream = [tokeniser tokenise:@"/* blah\nblah blah\n blah */ long jam\n\nlong ham"];
+    NSUInteger tokenLines[]   = {0, 2, 2, 2 , 2 , 2 , 4, 4, 4, 4};
+    NSUInteger tokenColumns[] = {0, 8, 9, 13, 14, 17, 0, 4, 5, 8};
+    NSUInteger tokenNumber = 0;
+    CPToken *token = nil;
+    while ((token = [tokenStream popToken]))
+    {
+        STAssertEquals([token lineNumber  ], tokenLines  [tokenNumber], @"Line number for token %lu is incorrect", tokenNumber, nil);
+        STAssertEquals([token columnNumber], tokenColumns[tokenNumber], @"Column number for toen %lu is incorrect", tokenNumber, nil);
+        tokenNumber++;
+    }
+}
+
 - (void)testMapCSSTokenisation
 {
     [self setUpMapCSS];