Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Tokenizer gets a lowercase option

  • Loading branch information...
commit b6ea405731eb699ee0c6396ab880c14df75f1d24 1 parent 298e2bd
@lok authored
Showing with 19 additions and 1 deletion.
  1. +2 −0  src/BKTokenizer.h
  2. +16 −1 src/BKTokenizer.m
  3. +1 −0  src/BKTokenizing.h
View
2  src/BKTokenizer.h
@@ -41,7 +41,9 @@
@interface BKTokenizer : NSObject <BKTokenizing> {
+ BOOL lowerCaseTokens;
}
+@property (readwrite) BOOL lowerCaseTokens;
- (NSArray*)tokenizeString:(NSString *)string;
View
17 src/BKTokenizer.m
@@ -40,6 +40,17 @@
@implementation BKTokenizer
+@synthesize lowerCaseTokens;
+
+- (id)init
+{
+ self = [super init];
+ if (self) {
+ lowerCaseTokens = YES;
+ }
+ return self;
+}
+
- (NSArray*)tokenizeString:(NSString *)string
{
PKTokenizer *tokenizer = [PKTokenizer tokenizerWithString:string];
@@ -50,7 +61,11 @@ - (NSArray*)tokenizeString:(NSString *)string
while ((token = [tokenizer nextToken]) != eof) {
if ([token tokenType] == PKTokenTypeWord || [token tokenType] == PKTokenTypeSymbol) {
- [tokens addObject:[token stringValue]];
+ NSString *tokenString = [token stringValue];
+ if (lowerCaseTokens) {
+ tokenString = [tokenString lowercaseString];
+ }
+ [tokens addObject:tokenString];
}
}
View
1  src/BKTokenizing.h
@@ -40,6 +40,7 @@
@protocol BKTokenizing
+@property (readwrite) BOOL lowerCaseTokens;
- (NSArray*)tokenizeString:(NSString*)string;
@end
Please sign in to comment.
Something went wrong with that request. Please try again.