Permalink
Browse files

Optimise the keyword recogniser, and provide a new method of ignoring…

…/adding tokens going onto the output stream, saving a lot of allocation of arrays.
  • Loading branch information...
1 parent 39c3e31 commit 741ffaea668d635071c69ffbbbafbd301049d2b3 @beelsebob committed Aug 23, 2012
@@ -18,6 +18,8 @@
*/
@protocol CPTokeniserDelegate <NSObject>
+@required
+
/**
* Determines whether a CPTokeniser should produce a token and consume the associated input string.
*
@@ -29,16 +31,32 @@
*/
- (BOOL)tokeniser:(CPTokeniser *)tokeniser shouldConsumeToken:(CPToken *)token;
+@optional
+
/**
* Allows you to replace a taken in the tokeniser's output stream.
*
+ * Only one of this and tokeniser:willProduceToken: will be called. The tokeniser will attempt to call this method second.
+ *
* @param tokeniser The CPTokeniser that will produce the token.
* @param token The CPToken that the tokeniser has recognised.
* @return Return an array of CPToken objects to place in the output token stream.
+ *
+ * @bug Warning this method is deprecated, use -tokeniser:requestsToken:pushedOntoStream: instead.
+ * @see tokeniser:requestsToken:pushedOntoStream:
*/
-- (NSArray *)tokeniser:(CPTokeniser *)tokeniser willProduceToken:(CPToken *)token;
+- (NSArray *)tokeniser:(CPTokeniser *)tokeniser willProduceToken:(CPToken *)token __attribute__((deprecated("Use tokeniser:requestsToken:pushedOntoStream: instead")));
-@optional
+/**
+ * Requests that you push a token onto a tokeniser's output stream. This allows you to replace the token with any others you choose, or not output the token at all.
+ *
+ * Only one of this and tokeniser:willProduceToken: will be called. The tokeniser will attempt to call this method first.
+ *
+ * @param tokeniser The CPTokeniser that produced the token.
+ * @param token The CPToken that the tokeniser has recognised.
+ * @param stream The CPTokenStream the token should be pushed onto (if required).
+ */
+- (void)tokeniser:(CPTokeniser *)tokeniser requestsToken:(CPToken *)token pushedOntoStream:(CPTokenStream *)stream;
/**
* This method is called when no recogniser matches a token at the current position in the input stream. You must provide a new location in the input stream to start
@@ -14,6 +14,7 @@
typedef struct
{
unsigned int shouldConsumeToken:1;
+ unsigned int requestsPush:1;
unsigned int willProduceToken:1;
unsigned int didNotFindTokenOnInputPositionError:1;
@@ -190,16 +191,18 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
- (void)addToken:(CPToken *)tok toStream:(CPTokenStream *)stream
{
- NSArray *toks;
- if (delegateRespondsTo.willProduceToken)
+ if (delegateRespondsTo.requestsPush)
{
- toks = [delegate tokeniser:self willProduceToken:tok];
+ [delegate tokeniser:self requestsToken:tok pushedOntoStream:stream];
+ }
+ else if (delegateRespondsTo.willProduceToken)
+ {
+ [stream pushTokens:[delegate tokeniser:self willProduceToken:tok]];
}
else
{
- toks = [NSArray arrayWithObject:tok];
+ [stream pushToken:tok];
}
- [stream pushTokens:toks];
}
- (void)advanceLineNumber:(NSUInteger *)ln columnNumber:(NSUInteger *)cn withInput:(NSString *)input range:(NSRange)range
@@ -232,6 +235,7 @@ - (void)setDelegate:(id<CPTokeniserDelegate>)aDelegate
delegate = aDelegate;
delegateRespondsTo.shouldConsumeToken = [delegate respondsToSelector:@selector(tokeniser:shouldConsumeToken:)];
+ delegateRespondsTo.requestsPush = [delegate respondsToSelector:@selector(tokeniser:requestsToken:pushedOntoStream:)];
delegateRespondsTo.willProduceToken = [delegate respondsToSelector:@selector(tokeniser:willProduceToken:)];
delegateRespondsTo.didNotFindTokenOnInputPositionError = [delegate respondsToSelector:@selector(tokeniser:didNotFindTokenOnInput:position:error:)];
}
@@ -85,19 +85,18 @@ - (void)dealloc
- (CPToken *)recogniseTokenInString:(NSString *)tokenString currentTokenPosition:(NSUInteger *)tokenPosition
{
- NSString *kw = [self keyword];
- NSUInteger kwLength = [kw length];
+ NSUInteger kwLength = [keyword length];
NSUInteger remainingChars = [tokenString length] - *tokenPosition;
if (remainingChars >= kwLength)
{
- if ([[tokenString substringWithRange:NSMakeRange(*tokenPosition, kwLength)] isEqualToString:kw])
+ if ([tokenString rangeOfString:keyword options:NSAnchoredSearch | NSLiteralSearch range:NSMakeRange(*tokenPosition, kwLength)].location != NSNotFound)
{
if (remainingChars == kwLength ||
nil == invalidFollowingCharacters ||
- [tokenString rangeOfCharacterFromSet:invalidFollowingCharacters options:0x0 range:NSMakeRange(*tokenPosition + kwLength, 1)].location == NSNotFound)
+ [tokenString rangeOfCharacterFromSet:invalidFollowingCharacters options:NSAnchoredSearch range:NSMakeRange(*tokenPosition + kwLength, 1)].location == NSNotFound)
{
*tokenPosition = *tokenPosition + kwLength;
- return [CPKeywordToken tokenWithKeyword:kw];
+ return [CPKeywordToken tokenWithKeyword:keyword];
}
}
}
@@ -66,34 +66,26 @@ - (BOOL)tokeniser:(CPTokeniser *)tokeniser shouldConsumeToken:(CPToken *)token
return YES;
}
-- (NSArray *)tokeniser:(CPTokeniser *)tokeniser willProduceToken:(CPToken *)token
+- (void)tokeniser:(CPTokeniser *)tokeniser requestsToken:(CPToken *)token pushedOntoStream:(CPTokenStream *)stream
{
- NSString *name = [token name];
if ([token isWhiteSpaceToken])
{
if (justTokenisedObject)
{
- return [NSArray arrayWithObject:token];
- }
- else
- {
- return [NSArray array];
+ [stream pushToken:token];
}
}
-
- justTokenisedObject = NO;
- if ([name isEqualToString:@"Comment"])
- {
- return [NSArray array];
- }
-
- if ([name isEqualToString:@"node"] || [name isEqualToString:@"way" ] || [name isEqualToString:@"relation"] ||
- [name isEqualToString:@"area"] || [name isEqualToString:@"line"] || [name isEqualToString:@"canvas"] || [name isEqualToString:@"*"])
+ else
{
- justTokenisedObject = YES;
+ NSString *name = [token name];
+ justTokenisedObject = ([name isEqualToString:@"node"] || [name isEqualToString:@"way" ] || [name isEqualToString:@"relation"] ||
+ [name isEqualToString:@"area"] || [name isEqualToString:@"line"] || [name isEqualToString:@"canvas"] || [name isEqualToString:@"*"]);
+
+ if (![name isEqualToString:@"Comment"])
+ {
+ [stream pushToken:token];
+ }
}
-
- return [NSArray arrayWithObject:token];
}
@end
@@ -311,7 +311,7 @@ - (void)testTokenLineColumnNumbers
{
STAssertEquals([token lineNumber ], tokenLines [tokenNumber] , @"Line number for token %lu is incorrect", tokenNumber, nil);
STAssertEquals([token columnNumber ], tokenColumns[tokenNumber] , @"Column number for token %lu is incorrect", tokenNumber, nil);
- STAssertEquals([token characterNumber], tokenPositions[tokenNumber], @"Character nmber for token %lu is incorrect", tokenNumber, nil);
+ STAssertEquals([token characterNumber], tokenPositions[tokenNumber], @"Character number for token %lu is incorrect", tokenNumber, nil);
tokenNumber++;
}
}

0 comments on commit 741ffae

Please sign in to comment.