Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Use parseKit as default Tokenizer

  • Loading branch information...
commit 8085e0a6097cf74774bd12f0dc56eabbb4becaa4 1 parent a483b15
@lok authored
Showing with 14 additions and 8 deletions.
  1. +1 −3 src/BKTokenizer.h
  2. +13 −4 src/BKTokenizer.m
  3. +0 −1  src/BKTokenizing.h
View
4 src/BKTokenizer.h
@@ -37,14 +37,12 @@
#import <Foundation/Foundation.h>
#import <BayesianKit/BKTokenizing.h>
+#import <ParseKit/ParseKit.h>
@interface BKTokenizer : NSObject <BKTokenizing> {
- BOOL lower;
}
-@property (readwrite, assign) BOOL lower;
-
- (NSArray*)tokenizeString:(NSString *)string;
@end
View
17 src/BKTokenizer.m
@@ -40,12 +40,21 @@
@implementation BKTokenizer
-@synthesize lower;
-
- (NSArray*)tokenizeString:(NSString *)string
{
- NSCharacterSet *set = [NSCharacterSet characterSetWithCharactersInString:@" \n;,()[]{}"];
- return [string componentsSeparatedByCharactersInSet:set];
+ PKTokenizer *tokenizer = [PKTokenizer tokenizerWithString:string];
+
+ PKToken *eof = [PKToken EOFToken];
+ PKToken *token = nil;
+ NSMutableSet *tokens = [NSMutableSet set];
+
+ while ((token = [tokenizer nextToken]) != eof) {
+ if ([token tokenType] == PKTokenTypeWord || [token tokenType] == PKTokenTypeSymbol) {
+ [tokens addObject:[token stringValue]];
+ }
+ }
+
+ return [tokens allObjects];
}
@end
View
1  src/BKTokenizing.h
@@ -40,7 +40,6 @@
@protocol BKTokenizing
-@property (readwrite, assign) BOOL lower;
- (NSArray*)tokenizeString:(NSString*)string;
@end
Please sign in to comment.
Something went wrong with that request. Please try again.