Merge pull request #1 from beelsebob/master

Merge in upstream
jbrennan · Sep 7, 2014 · 99790e1 · 99790e1
2 parents e1c0747 + 2dba13f
commit 99790e1
Show file tree

Hide file tree

Showing 23 changed files with 368 additions and 16 deletions.
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+/.DS_Store
+
+/CoreParse.xcodeproj/xcuserdata/
+
+/CoreParse.xcodeproj/project.xcworkspace/xcuserdata/
diff --git a/CoreParse.xcodeproj/project.pbxproj b/CoreParse.xcodeproj/project.pbxproj
@@ -167,8 +167,19 @@
 		1FC18282139AE3810027F597 /* CPGrammarInternal.h in Headers */ = {isa = PBXBuildFile; fileRef = 1FC18280139AE3800027F597 /* CPGrammarInternal.h */; };
 		1FC18283139AE3810027F597 /* CPGrammarInternal.m in Sources */ = {isa = PBXBuildFile; fileRef = 1FC18281139AE3810027F597 /* CPGrammarInternal.m */; };
 		1FC18287139BA47D0027F597 /* CoreParse-template.md in Resources */ = {isa = PBXBuildFile; fileRef = 1FC18286139BA47D0027F597 /* CoreParse-template.md */; };
+		54197209188F7D49004240B4 /* CPRegexpRecogniser.h in Headers */ = {isa = PBXBuildFile; fileRef = 54197207188F7D49004240B4 /* CPRegexpRecogniser.h */; };
+		5419720A188F7D49004240B4 /* CPRegexpRecogniser.h in Headers */ = {isa = PBXBuildFile; fileRef = 54197207188F7D49004240B4 /* CPRegexpRecogniser.h */; };
+		5419720B188F7D49004240B4 /* CPRegexpRecogniser.m in Sources */ = {isa = PBXBuildFile; fileRef = 54197208188F7D49004240B4 /* CPRegexpRecogniser.m */; };
+		5419720D188F7D49004240B4 /* CPRegexpRecogniser.m in Sources */ = {isa = PBXBuildFile; fileRef = 54197208188F7D49004240B4 /* CPRegexpRecogniser.m */; };
+		5419721D188F9A54004240B4 /* CPRegexpRecogniserTest.m in Sources */ = {isa = PBXBuildFile; fileRef = 5419721C188F9A54004240B4 /* CPRegexpRecogniserTest.m */; };
 		DA2B1DF81566B704002FDBD7 /* CPSTAssertionsTests.m in Sources */ = {isa = PBXBuildFile; fileRef = DA2B1DF71566B704002FDBD7 /* CPSTAssertionsTests.m */; };
 		DA2B1DF91566B704002FDBD7 /* CPSTAssertionsTests.m in Sources */ = {isa = PBXBuildFile; fileRef = DA2B1DF71566B704002FDBD7 /* CPSTAssertionsTests.m */; };
+		DC3E9BEF191DBAC800F6023C /* CPWillFinishDelegateTest.m in Sources */ = {isa = PBXBuildFile; fileRef = DC3E9BEE191DBAC800F6023C /* CPWillFinishDelegateTest.m */; };
+		EA5FF42E1884CC6600BEEF03 /* CPTestErrorEvaluatorDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 1F893A2214DEF40D00316FF7 /* CPTestErrorEvaluatorDelegate.m */; };
+		EA5FF42F1884CC7100BEEF03 /* CPTestErrorHandlingDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = 1FA68DA414DE9D3D005519B9 /* CPTestErrorHandlingDelegate.m */; };
+		EA5FF4301884FFDC00BEEF03 /* Expression2.m in Sources */ = {isa = PBXBuildFile; fileRef = 1FA4386C162F515C00B92704 /* Expression2.m */; };
+		EA5FF4311884FFE200BEEF03 /* RuleBase.m in Sources */ = {isa = PBXBuildFile; fileRef = 1FA4386E162F515C00B92704 /* RuleBase.m */; };
+		EA5FF4321884FFE500BEEF03 /* Term2.m in Sources */ = {isa = PBXBuildFile; fileRef = 1FA43870162F515C00B92704 /* Term2.m */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXContainerItemProxy section */
@@ -311,9 +322,13 @@
 		1FC18286139BA47D0027F597 /* CoreParse-template.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = "CoreParse-template.md"; sourceTree = "<group>"; };
 		1FE77D551375AE6F00879A41 /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = System/Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; };
 		1FE77D821375EA8F00879A41 /* CoreParse Documentation */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.executable"; includeInIndex = 0; path = "CoreParse Documentation"; sourceTree = BUILT_PRODUCTS_DIR; };
+		54197207188F7D49004240B4 /* CPRegexpRecogniser.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CPRegexpRecogniser.h; sourceTree = "<group>"; };
+		54197208188F7D49004240B4 /* CPRegexpRecogniser.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CPRegexpRecogniser.m; sourceTree = "<group>"; };
+		5419721C188F9A54004240B4 /* CPRegexpRecogniserTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CPRegexpRecogniserTest.m; sourceTree = "<group>"; };
 		DA2B1DF41566B311002FDBD7 /* CPSenTestKitAssertions.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = CPSenTestKitAssertions.h; sourceTree = "<group>"; };
 		DA2B1DF61566B704002FDBD7 /* CPSTAssertionsTests.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CPSTAssertionsTests.h; sourceTree = "<group>"; };
 		DA2B1DF71566B704002FDBD7 /* CPSTAssertionsTests.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CPSTAssertionsTests.m; sourceTree = "<group>"; };
+		DC3E9BEE191DBAC800F6023C /* CPWillFinishDelegateTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = CPWillFinishDelegateTest.m; sourceTree = "<group>"; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -448,6 +463,8 @@
 				1FB81325132FF16E0095982D /* CPTestMapCSSTokenisingDelegate.m */,
 				1FA68DA314DE9D3D005519B9 /* CPTestErrorHandlingDelegate.h */,
 				1FA68DA414DE9D3D005519B9 /* CPTestErrorHandlingDelegate.m */,
+				5419721C188F9A54004240B4 /* CPRegexpRecogniserTest.m */,
+				DC3E9BEE191DBAC800F6023C /* CPWillFinishDelegateTest.m */,
 				1F9F839F13B731F3006E939D /* Expression.h */,
 				1F9F83A013B731F3006E939D /* Expression.m */,
 				1F9F83A413B732AC006E939D /* Term.h */,
@@ -559,6 +576,8 @@
 				1F1DFCE01306E3C300B22855 /* CPIdentifierRecogniser.m */,
 				1F1DFCF11307D2C500B22855 /* CPQuotedRecogniser.h */,
 				1F1DFCF21307D2C500B22855 /* CPQuotedRecogniser.m */,
+				54197207188F7D49004240B4 /* CPRegexpRecogniser.h */,
+				54197208188F7D49004240B4 /* CPRegexpRecogniser.m */,
 			);
 			path = "Token Recognisers";
 			sourceTree = "<group>";
@@ -632,6 +651,7 @@
 			files = (
 				1F0E8935130463D000537D04 /* CoreParse.h in Headers */,
 				1F0E8939130463E900537D04 /* CPTokeniser.h in Headers */,
+				54197209188F7D49004240B4 /* CPRegexpRecogniser.h in Headers */,
 				1F0E8943130466D000537D04 /* CPTokenStream.h in Headers */,
 				1F0E894C1306795800537D04 /* CPToken.h in Headers */,
 				1F4683891306AA8500407491 /* CPKeywordRecogniser.h in Headers */,
@@ -687,6 +707,7 @@
 			isa = PBXHeadersBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				5419720A188F7D49004240B4 /* CPRegexpRecogniser.h in Headers */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -921,6 +942,7 @@
 				1FB81327132FF16E0095982D /* CPTestMapCSSTokenisingDelegate.m in Sources */,
 				1F45A2E513422E1300092D78 /* CPJSONParser.m in Sources */,
 				1F6D44911348CC9600E982C7 /* CPLALR1Parser.m in Sources */,
+				5419720B188F7D49004240B4 /* CPRegexpRecogniser.m in Sources */,
 				1FC1827E139ADC800027F597 /* CPGrammarPrivate.m in Sources */,
 				1FC18283139AE3810027F597 /* CPGrammarInternal.m in Sources */,
 				1F9F83AD13B7CABA006E939D /* CPRHSItem.m in Sources */,
@@ -947,6 +969,8 @@
 				1FA798261567DC58003AC8AE /* CPTestMapCSSTokenisingDelegate.m in Sources */,
 				1FA798271567DC5C003AC8AE /* CPTestErrorHandlingDelegate.m in Sources */,
 				1FA43871162F515C00B92704 /* Expression2.m in Sources */,
+				DC3E9BEF191DBAC800F6023C /* CPWillFinishDelegateTest.m in Sources */,
+				5419721D188F9A54004240B4 /* CPRegexpRecogniserTest.m in Sources */,
 				1FA43872162F515C00B92704 /* RuleBase.m in Sources */,
 				1FA43873162F515C00B92704 /* Term2.m in Sources */,
 			);
@@ -980,6 +1004,7 @@
 				1F9281B7145C11A60033BC34 /* CPGrammarSymbol.m in Sources */,
 				1F9281B8145C11A60033BC34 /* CPRHSItem.m in Sources */,
 				1F9281B9145C11A60033BC34 /* CPRHSItemResult.m in Sources */,
+				5419720D188F7D49004240B4 /* CPRegexpRecogniser.m in Sources */,
 				1F9281BA145C11A60033BC34 /* CPSyntaxTree.m in Sources */,
 				1F9281BB145C11A60033BC34 /* CPParser.m in Sources */,
 				1F9281BC145C11A60033BC34 /* CPShiftReduceAction.m in Sources */,
@@ -1001,13 +1026,18 @@
 			isa = PBXSourcesBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				EA5FF4321884FFE500BEEF03 /* Term2.m in Sources */,
+				EA5FF42F1884CC7100BEEF03 /* CPTestErrorHandlingDelegate.m in Sources */,
 				1F9281C9145C18220033BC34 /* CPTestEvaluatorDelegate.m in Sources */,
 				1F9281CA145C18220033BC34 /* CPTestWhiteSpaceIgnoringDelegate.m in Sources */,
 				1F9281CB145C18220033BC34 /* CPTestMapCSSTokenisingDelegate.m in Sources */,
+				EA5FF4301884FFDC00BEEF03 /* Expression2.m in Sources */,
 				1F9281CC145C18220033BC34 /* Expression.m in Sources */,
+				EA5FF42E1884CC6600BEEF03 /* CPTestErrorEvaluatorDelegate.m in Sources */,
 				1F9281CD145C18220033BC34 /* Term.m in Sources */,
 				1F9281C8145C17580033BC34 /* CoreParseTests.m in Sources */,
 				DA2B1DF91566B704002FDBD7 /* CPSTAssertionsTests.m in Sources */,
+				EA5FF4311884FFE200BEEF03 /* RuleBase.m in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};

diff --git a/CoreParse.xcodeproj/project.xcworkspace/xcshareddata/CoreParse.xccheckout b/CoreParse.xcodeproj/project.xcworkspace/xcshareddata/CoreParse.xccheckout
@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDESourceControlProjectFavoriteDictionaryKey</key>
+	<false/>
+	<key>IDESourceControlProjectIdentifier</key>
+	<string>5FCEC138-BD89-4E15-965B-903D778E9B1D</string>
+	<key>IDESourceControlProjectName</key>
+	<string>CoreParse</string>
+	<key>IDESourceControlProjectOriginsDictionary</key>
+	<dict>
+		<key>933D72C4-BB34-468B-8A2A-F9A5F6A06A65</key>
+		<string>https://github.com/beelsebob/CoreParse.git</string>
+	</dict>
+	<key>IDESourceControlProjectPath</key>
+	<string>CoreParse.xcodeproj/project.xcworkspace</string>
+	<key>IDESourceControlProjectRelativeInstallPathDictionary</key>
+	<dict>
+		<key>933D72C4-BB34-468B-8A2A-F9A5F6A06A65</key>
+		<string>../..</string>
+	</dict>
+	<key>IDESourceControlProjectURL</key>
+	<string>ssh://github.com/siuying/CSSSelectorConverter.git</string>
+	<key>IDESourceControlProjectVersion</key>
+	<integer>110</integer>
+	<key>IDESourceControlProjectWCCIdentifier</key>
+	<string>933D72C4-BB34-468B-8A2A-F9A5F6A06A65</string>
+	<key>IDESourceControlProjectWCConfigurations</key>
+	<array>
+		<dict>
+			<key>IDESourceControlRepositoryExtensionIdentifierKey</key>
+			<string>public.vcs.git</string>
+			<key>IDESourceControlWCCIdentifierKey</key>
+			<string>933D72C4-BB34-468B-8A2A-F9A5F6A06A65</string>
+			<key>IDESourceControlWCCName</key>
+			<string>CoreParse</string>
+		</dict>
+	</array>
+</dict>
+</plist>
diff --git a/CoreParse/CoreParse.h b/CoreParse/CoreParse.h
@@ -16,6 +16,7 @@
 #import "CPWhitespaceRecogniser.h"
 #import "CPIdentifierRecogniser.h"
 #import "CPQuotedRecogniser.h"
+#import "CPRegexpRecogniser.h"
 
 #import "CPToken.h"
 #import "CPErrorToken.h"

diff --git a/CoreParse/Grammar/CPGrammar.m b/CoreParse/Grammar/CPGrammar.m
@@ -444,7 +444,7 @@ - (NSArray *)rulesForNonTerminalWithName:(NSString *)nonTerminal
 
 - (NSUInteger)hash
 {
-    return [[self start] hash] << 16 + [[self rules] hash];
+    return [[self start] hash] ^ [[self rules] hash];
 }
 
 - (BOOL)isGrammar

diff --git a/CoreParse/Grammar/CPGrammarInternal.m b/CoreParse/Grammar/CPGrammarInternal.m
@@ -84,7 +84,7 @@ - (NSSet *)lr0GotoKernelWithItems:(NSSet *)i symbol:(CPGrammarSymbol *)symbol
              {
                  return [symbol isEqualToGrammarSymbol:[item nextSymbol]];
              }]
-            map:^ id (CPItem *item)
+            cp_map:^ id (CPItem *item)
             {
                 return [item itemByMovingDotRight];
             }];
@@ -101,7 +101,7 @@ - (NSArray *)lr0Kernels
     {
         NSSet *kernel = [processingQueue objectAtIndex:0];
         NSSet *itemSet = [self lr0Closure:kernel];
-        NSSet *validNexts = [itemSet map:^ id (CPItem *item) { return [item nextSymbol]; }];
+        NSSet *validNexts = [itemSet cp_map:^ id (CPItem *item) { return [item nextSymbol]; }];
 
         for (CPGrammarSymbol *s in validNexts)
         {
@@ -163,7 +163,7 @@ - (NSSet *)lr1GotoKernelWithItems:(NSSet *)i symbol:(CPGrammarSymbol *)symbol
              {
                  return [symbol isEqualToGrammarSymbol:[item nextSymbol]];
              }]
-            map:^ id (CPItem *item)
+            cp_map:^ id (CPItem *item)
             {
                 return [item itemByMovingDotRight];
             }];

diff --git a/CoreParse/Grammar/CPRHSItem.m b/CoreParse/Grammar/CPRHSItem.m
@@ -23,7 +23,7 @@ @implementation CPRHSItem
 
 - (NSUInteger)hash
 {
-    return [[self alternatives] hash] << 2 + ([self repeats] ? 0x2 : 0x0) + ([self mayNotExist] ? 0x1 : 0x0);
+    return [[self alternatives] hash] ^ ([self repeats] ? 0x2 : 0x0) + ([self mayNotExist] ? 0x1 : 0x0);
 }
 
 - (BOOL)isRHSItem

diff --git a/CoreParse/Grammar/CPRule+Internal.h b/CoreParse/Grammar/CPRule+Internal.h
@@ -6,7 +6,7 @@
 //  Copyright (c) 2012 In The Beginning... All rights reserved.
 //
 
-#import <CoreParse/CoreParse.h>
+#import "CoreParse.h"
 
 @interface CPRule (Internal)
 

diff --git a/CoreParse/Grammar/CPRule.m b/CoreParse/Grammar/CPRule.m
@@ -134,7 +134,7 @@ - (NSString *)description
 
 - (NSUInteger)hash
 {
-    return [name hash] << 16 + [self tag] ;
+    return [name hash] ^ [self tag];
 }
 
 - (BOOL)isRule

diff --git a/CoreParse/NSArray+Functional.h b/CoreParse/NSArray+Functional.h
@@ -10,6 +10,6 @@
 
 @interface NSArray (Functional)
 
-- (NSArray *)map:(id(^)(id obj))block;
+- (NSArray *)cp_map:(id(^)(id obj))block;
 
 @end
diff --git a/CoreParse/NSArray+Functional.m b/CoreParse/NSArray+Functional.m
@@ -10,7 +10,7 @@
 
 @implementation NSArray (Functional)
 
-- (NSArray *)map:(id(^)(id obj))block
+- (NSArray *)cp_map:(id(^)(id obj))block
 {
     NSUInteger c = [self count];
     id *resultingObjects = malloc(c * sizeof(id));

diff --git a/CoreParse/NSSetFunctional.h b/CoreParse/NSSetFunctional.h
@@ -11,6 +11,6 @@
 
 @interface NSSet(Functional)
 
-- (NSSet *)map:(id(^)(id obj))block;
+- (NSSet *)cp_map:(id(^)(id obj))block;
 
 @end
diff --git a/CoreParse/NSSetFunctional.m b/CoreParse/NSSetFunctional.m
@@ -11,7 +11,7 @@
 
 @implementation NSSet(Functional)
 
-- (NSSet *)map:(id(^)(id obj))block
+- (NSSet *)cp_map:(id(^)(id obj))block
 {
     NSUInteger c = [self count];
     id *resultingObjects = malloc(c * sizeof(id));

diff --git a/CoreParse/Parsers/CPShiftReduceParsers/CPLALR1Parser.m b/CoreParse/Parsers/CPShiftReduceParsers/CPLALR1Parser.m
@@ -30,7 +30,7 @@ - (BOOL)constructShiftReduceTables
 {
     CPGrammar *aug = [[self grammar] augmentedGrammar];
     NSArray *kernels = [self kernelsForGrammar:aug];
-    NSArray *lr0Kernels = [kernels map:^ NSSet * (NSSet *s) { return [s map:^ id (CPLR1Item *i) { return [CPItem itemWithRule:[i rule] position:[i position]]; }]; }];
+    NSArray *lr0Kernels = [kernels cp_map:^ NSSet * (NSSet *s) { return [s cp_map:^ id (CPLR1Item *i) { return [CPItem itemWithRule:[i rule] position:[i position]]; }]; }];
     NSUInteger itemCount = [kernels count];
     NSArray *allNonTerminalNames = [[self grammar] allNonTerminalNames];
     NSString *startSymbol = [aug start];
@@ -73,7 +73,7 @@ - (BOOL)constructShiftReduceTables
                     else if ([next isTerminal])
                     {
                         NSSet *g = [aug lr0GotoKernelWithItems:itemsSet symbol:next];
-                        NSSet *lr0G = [g map:^ id (CPLR1Item *i) { return [CPItem itemWithRule:[i rule] position:[i position]]; }];
+                        NSSet *lr0G = [g cp_map:^ id (CPLR1Item *i) { return [CPItem itemWithRule:[i rule] position:[i position]]; }];
                         NSUInteger indx = 0;
                         NSUInteger ix = NSNotFound;
                         for (NSSet *lr0Kernel in lr0Kernels)
@@ -98,7 +98,7 @@ - (BOOL)constructShiftReduceTables
             for (NSString *nonTerminalName in allNonTerminalNames)
             {
                 NSSet *g = [aug lr0GotoKernelWithItems:itemsSet symbol:[CPGrammarSymbol nonTerminalWithName:nonTerminalName]];
-                NSSet *lr0G = [g map:^ id (CPLR1Item *i) { return [CPItem itemWithRule:[i rule] position:[i position]]; }];
+                NSSet *lr0G = [g cp_map:^ id (CPLR1Item *i) { return [CPItem itemWithRule:[i rule] position:[i position]]; }];
                 NSUInteger indx = 0;
                 NSUInteger gotoIndex = NSNotFound;
                 for (NSSet *lr0Kernel in lr0Kernels)

diff --git a/CoreParse/Parsers/CPShiftReduceParsers/CPLR1Item.m b/CoreParse/Parsers/CPShiftReduceParsers/CPLR1Item.m
@@ -66,7 +66,7 @@ - (BOOL)isEqual:(id)object
 
 - (NSUInteger)hash
 {
-    return [[self rule] hash] << 16 + [terminal hash] + [self position];
+    return [[self rule] hash] ^ [terminal hash] ^ [self position];
 }
 
 - (NSString *)description

diff --git a/CoreParse/Parsers/CPShiftReduceParsers/CPLR1Parser.m b/CoreParse/Parsers/CPShiftReduceParsers/CPLR1Parser.m
@@ -104,7 +104,7 @@ - (NSArray *)kernelsForGrammar:(CPGrammar *)aug
     {
         NSSet *kernels = [processingQueue objectAtIndex:0];
         NSSet *itemSet = [aug lr1Closure:kernels];
-        NSSet *validNexts = [itemSet map:^ id (CPItem *item)
+        NSSet *validNexts = [itemSet cp_map:^ id (CPItem *item)
                              {
                                  return [item nextSymbol];
                              }];

diff --git a/CoreParse/Tokenisation/CPTokeniser.h b/CoreParse/Tokenisation/CPTokeniser.h
@@ -70,6 +70,17 @@
  */
 - (NSUInteger)tokeniser:(CPTokeniser *)tokeniser didNotFindTokenOnInput:(NSString *)input position:(NSUInteger)position error:(NSString **)errorMessage;
 
+/**
+ * This method is called when the tokeniser has finished tokenising and is about to push the
+ * <EOF> token onto the token stream. This gives the delegate a chance to push additional tokens
+ * (such as scope closing tokens in a python tokeniser) onto the token stream before the <EOF>
+ * token is pushed onto the stream.
+ *
+ * @param tokeniser The CPTokeniser that is finishing
+ * @param stream the CPTokenStream any tokens should be pushed onto (if required).
+ */
+- (void)tokeniserWillFinish:(CPTokeniser *)tokeniser stream:(CPTokenStream *)stream;
+
 @end
 
 /**

diff --git a/CoreParse/Tokenisation/CPTokeniser.m b/CoreParse/Tokenisation/CPTokeniser.m
@@ -18,6 +18,7 @@
     unsigned int requestsPush:1;
     unsigned int willProduceToken:1;
     unsigned int didNotFindTokenOnInputPositionError:1;
+    unsigned int willFinish:1;
 
 } CPTokeniserDelegateResponseCache;
 
@@ -189,6 +190,11 @@ - (void)tokenise:(NSString *)input into:(CPTokenStream *)stream
     }
     if (inputLength <= currentTokenOffset)
     {
+        if (delegateRespondsTo.willFinish)
+        {
+            [delegate tokeniserWillFinish:self stream:stream];
+        }
+
         CPEOFToken *token = [CPEOFToken eof];
         [token setLineNumber:currentLineNumber];
         [token setColumnNumber:currentColumnNumber];
@@ -257,6 +263,7 @@ - (void)setDelegate:(id<CPTokeniserDelegate>)aDelegate
         delegateRespondsTo.requestsPush = [delegate respondsToSelector:@selector(tokeniser:requestsToken:pushedOntoStream:)];
         delegateRespondsTo.willProduceToken = [delegate respondsToSelector:@selector(tokeniser:willProduceToken:)];
         delegateRespondsTo.didNotFindTokenOnInputPositionError = [delegate respondsToSelector:@selector(tokeniser:didNotFindTokenOnInput:position:error:)];
+        delegateRespondsTo.willFinish = [delegate respondsToSelector:@selector(tokeniserWillFinish:stream:)];
     }
 }