Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Added character position to token data structure.

  • Loading branch information...
commit 287b681eabd6b1e9f628a3af3b2425a41e52bc37 1 parent 32dc81b
@beelsebob beelsebob authored
View
2  CoreParse/Tokenisation/CPTokeniser.m
@@ -119,6 +119,7 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
{
[tok setLineNumber:currentLineNumber];
[tok setColumnNumber:currentColumnNumber];
+ [tok setCharacterNumber:lastTokenOffset];
if ([delegate respondsToSelector:@selector(tokeniser:shouldConsumeToken:)])
{
@@ -169,6 +170,7 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
CPEOFToken *token = [CPEOFToken eof];
[token setLineNumber:currentLineNumber];
[token setColumnNumber:currentColumnNumber];
+ [token setCharacterNumber:inputLength];
[stream pushToken:token];
}
[stream closeTokenStream];
View
5 CoreParse/Tokenisation/Token Types/CPToken.h
@@ -32,4 +32,9 @@
*/
@property (readwrite, assign) NSUInteger columnNumber;
+/**
+ * The index in the input string of the first character in this token.
+ */
+@property (readwrite, assign) NSUInteger characterNumber;
+
@end
View
1  CoreParse/Tokenisation/Token Types/CPToken.m
@@ -12,6 +12,7 @@ @implementation CPToken
@synthesize lineNumber;
@synthesize columnNumber;
+@synthesize characterNumber;
- (NSString *)name
{
View
10 CoreParseTests/CoreParseTests.m
@@ -301,14 +301,16 @@ - (void)testTokenLineColumnNumbers
[tokeniser addTokenRecogniser:[CPIdentifierRecogniser identifierRecogniser]];
[tokeniser addTokenRecogniser:[CPWhiteSpaceRecogniser whiteSpaceRecogniser]];
CPTokenStream *tokenStream = [tokeniser tokenise:@"/* blah\nblah blah\n blah */ long jam\n\nlong ham"];
- NSUInteger tokenLines[] = {0, 2, 2, 2 , 2 , 2 , 4, 4, 4, 4};
- NSUInteger tokenColumns[] = {0, 8, 9, 13, 14, 17, 0, 4, 5, 8};
+ NSUInteger tokenLines[] = {0, 2 , 2 , 2 , 2 , 2 , 4 , 4 , 4 , 4 };
+ NSUInteger tokenColumns[] = {0, 8 , 9 , 13, 14, 17, 0 , 4 , 5 , 8 };
+ NSUInteger tokenPositions[] = {0, 26, 27, 31, 32, 35, 37, 41, 42, 45};
NSUInteger tokenNumber = 0;
CPToken *token = nil;
while ((token = [tokenStream popToken]))
{
- STAssertEquals([token lineNumber ], tokenLines [tokenNumber], @"Line number for token %lu is incorrect", tokenNumber, nil);
- STAssertEquals([token columnNumber], tokenColumns[tokenNumber], @"Column number for toen %lu is incorrect", tokenNumber, nil);
+ STAssertEquals([token lineNumber ], tokenLines [tokenNumber] , @"Line number for token %lu is incorrect", tokenNumber, nil);
+ STAssertEquals([token columnNumber ], tokenColumns[tokenNumber] , @"Column number for token %lu is incorrect", tokenNumber, nil);
+ STAssertEquals([token characterNumber], tokenPositions[tokenNumber], @"Character nmber for token %lu is incorrect", tokenNumber, nil);
tokenNumber++;
}
}
Please sign in to comment.
Something went wrong with that request. Please try again.