Permalink
Browse files

Added line and column numbers, fixes #12

  • Loading branch information...
1 parent 7c941b8 commit 45368c8157b7daf81892cb748b99e07172453532 @beelsebob beelsebob committed Feb 5, 2012
@@ -19,6 +19,7 @@ @interface CPTokeniser ()
@property (readwrite, retain) NSMutableArray *tokenRecognisers;
- (void)addToken:(CPToken *)tok toStream:(CPTokenStream *)stream;
+- (void)advanceLineNumber:(NSUInteger *)ln columnNumber:(NSUInteger *)cn withInput:(NSString *)input range:(NSRange)range;
@end
@@ -102,6 +103,8 @@ - (CPTokenStream *)tokenise:(NSString *)input
- (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
{
NSUInteger currentTokenOffset = 0;
+ NSUInteger currentLineNumber = 0;
+ NSUInteger currentColumnNumber = 0;
NSUInteger inputLength = [input length];
NSArray *recs = [self tokenRecognisers];
@@ -114,11 +117,15 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
CPToken *tok = [recogniser recogniseTokenInString:input currentTokenPosition:&currentTokenOffset];
if (nil != tok)
{
+ [tok setLineNumber:currentLineNumber];
+ [tok setColumnNumber:currentColumnNumber];
+
if ([delegate respondsToSelector:@selector(tokeniser:shouldConsumeToken:)])
{
if ([delegate tokeniser:self shouldConsumeToken:tok])
{
[self addToken:tok toStream:stream];
+ [self advanceLineNumber:&currentLineNumber columnNumber:&currentColumnNumber withInput:input range:NSMakeRange(lastTokenOffset, currentTokenOffset - lastTokenOffset)];
recognised = YES;
break;
}
@@ -130,6 +137,7 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
else
{
[self addToken:tok toStream:stream];
+ [self advanceLineNumber:&currentLineNumber columnNumber:&currentColumnNumber withInput:input range:NSMakeRange(lastTokenOffset, currentTokenOffset - lastTokenOffset)];
recognised = YES;
break;
}
@@ -158,7 +166,10 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
}
if (inputLength <= currentTokenOffset)
{
- [stream pushToken:[CPEOFToken eof]];
+ CPEOFToken *token = [CPEOFToken eof];
+ [token setLineNumber:currentLineNumber];
+ [token setColumnNumber:currentColumnNumber];
+ [stream pushToken:token];
}
[stream closeTokenStream];
}
@@ -177,4 +188,27 @@ - (void)addToken:(CPToken *)tok toStream:(CPTokenStream *)stream
[stream pushTokens:toks];
}
+- (void)advanceLineNumber:(NSUInteger *)ln columnNumber:(NSUInteger *)cn withInput:(NSString *)input range:(NSRange)range
+{
+ NSRange searchRange = range;
+ NSUInteger rangeEnd = range.location + range.length;
+ NSRange foundRange = [input rangeOfCharacterFromSet:[NSCharacterSet characterSetWithCharactersInString:@"\n\r"] options:NSLiteralSearch range:searchRange];
+ NSUInteger lastNewLineLocation = NSNotFound;
+ while (foundRange.location != NSNotFound)
+ {
+ *ln += foundRange.length;
+ lastNewLineLocation = foundRange.location + foundRange.length;
+ searchRange = NSMakeRange(lastNewLineLocation, rangeEnd - lastNewLineLocation);
+ foundRange = [input rangeOfCharacterFromSet:[NSCharacterSet characterSetWithCharactersInString:@"\n\r"] options:NSLiteralSearch range:searchRange];
+ }
+ if (lastNewLineLocation != NSNotFound)
+ {
+ *cn = rangeEnd - lastNewLineLocation;
+ }
+ else
+ {
+ *cn += range.length;
+ }
+}
+
@end
@@ -22,4 +22,14 @@
*/
@property (readonly) NSString *name;
+/**
+ * The line on which the token can be found.
+ */
+@property (readwrite, assign) NSUInteger lineNumber;
+
+/**
+ * The column on which the token can be found.
+ */
+@property (readwrite, assign) NSUInteger columnNumber;
+
@end
@@ -10,6 +10,9 @@
@implementation CPToken
+@synthesize lineNumber;
+@synthesize columnNumber;
+
- (NSString *)name
{
[NSException raise:@"Abstract method called exception" format:@"CPToken is abstract, and should not have name called."];
@@ -293,6 +293,26 @@ - (void)testTokeniserError
STAssertEqualObjects(tokenStream, expectedTokenStream, @"Inserting error token and continuing according to delegate failed.", nil);
}
+- (void)testTokenLineColumnNumbers
+{
+ CPTokeniser *tokeniser = [[[CPTokeniser alloc] init] autorelease];
+ [tokeniser addTokenRecogniser:[CPQuotedRecogniser quotedRecogniserWithStartQuote:@"/*" endQuote:@"*/" name:@"Comment"]];
+ [tokeniser addTokenRecogniser:[CPKeywordRecogniser recogniserForKeyword:@"long"]];
+ [tokeniser addTokenRecogniser:[CPIdentifierRecogniser identifierRecogniser]];
+ [tokeniser addTokenRecogniser:[CPWhiteSpaceRecogniser whiteSpaceRecogniser]];
+ CPTokenStream *tokenStream = [tokeniser tokenise:@"/* blah\nblah blah\n blah */ long jam\n\nlong ham"];
+ NSUInteger tokenLines[] = {0, 2, 2, 2 , 2 , 2 , 4, 4, 4, 4};
+ NSUInteger tokenColumns[] = {0, 8, 9, 13, 14, 17, 0, 4, 5, 8};
+ NSUInteger tokenNumber = 0;
+ CPToken *token = nil;
+ while ((token = [tokenStream popToken]))
+ {
+ STAssertEquals([token lineNumber ], tokenLines [tokenNumber], @"Line number for token %lu is incorrect", tokenNumber, nil);
+ STAssertEquals([token columnNumber], tokenColumns[tokenNumber], @"Column number for toen %lu is incorrect", tokenNumber, nil);
+ tokenNumber++;
+ }
+}
+
- (void)testMapCSSTokenisation
{
[self setUpMapCSS];

0 comments on commit 45368c8

Please sign in to comment.