Permalink
Browse files

merging from SVN - new AST-based Factory

  • Loading branch information...
1 parent 18001aa commit e7e198438b78ed817a64f1ab4ad652c16a173ccf @itod committed Mar 23, 2013
Showing with 6,564 additions and 4,124 deletions.
  1. +1 −3 Info.plist
  2. +1 −3 JSParseKit-Info.plist
  3. +459 −135 ParseKit.xcodeproj/project.pbxproj
  4. +56 −36 debugapp/DebugAppDelegate.m
  5. +39 −27 debugapp/DebugAppMainMenu.xib
  6. +36 −3 docs/web/tokenization.html
  7. +5 −4 {test → include/ParseKit}/PKAST.h
  8. +2 −12 include/ParseKit/PKDelimitState.h
  9. +2 −2 include/ParseKit/PKEmpty.h
  10. +10 −27 include/ParseKit/PKParserFactory.h
  11. +1 −1 include/ParseKit/PKTokenizer.h
  12. +2 −2 res/css2_1.grammar
  13. +2 −0 res/erb.grammar
  14. +2 −2 res/html.grammar
  15. +211 −0 res/sqlite.grammar
  16. +5 −0 res/sqlite_input.txt
  17. +4 −4 res/xml.grammar
  18. +28 −12 src/PKAssembly.m
  19. +13 −0 src/PKBaseScope.h
  20. +52 −0 src/PKBaseScope.m
  21. +29 −0 src/PKBaseSymbol.h
  22. +100 −0 src/PKBaseSymbol.m
  23. +33 −0 src/PKBaseVisitor.h
  24. +119 −0 src/PKBaseVisitor.m
  25. +14 −0 src/PKBuiltInTypeSymbol.h
  26. +13 −0 src/PKBuiltInTypeSymbol.m
  27. +17 −0 src/PKDefinitionPhaseVisitor.h
  28. +211 −0 src/PKDefinitionPhaseVisitor.m
  29. +18 −0 src/PKDelimitDescriptor.h
  30. +67 −0 src/PKDelimitDescriptor.m
  31. +19 −0 src/PKDelimitDescriptorCollection.h
  32. +65 −0 src/PKDelimitDescriptorCollection.m
  33. +85 −94 src/PKDelimitState.m
  34. +13 −0 src/PKGlobalScope.h
  35. +57 −0 src/PKGlobalScope.m
  36. +15 −4 src/PKGrammarParser.h
  37. +197 −142 src/PKGrammarParser.m
  38. +13 −1 src/PKLiteral.m
  39. +13 −0 src/PKLocalScope.h
  40. +13 −0 src/PKLocalScope.m
  41. +0 −8 src/PKNumberState.m
  42. +541 −538 src/PKParserFactory.m
  43. +6 −1 src/PKPattern.m
  44. +14 −0 src/PKResolutionPhaseVisitor.h
  45. +303 −0 src/PKResolutionPhaseVisitor.m
  46. +20 −0 src/PKScope.h
  47. +13 −3 src/PKSymbol.m
  48. +13 −0 src/PKSymbolTable.h
  49. +52 −0 src/PKSymbolTable.m
  50. +27 −0 src/PKTokenizer.m
  51. +1 −1 src/PKTrack.m
  52. +14 −0 src/PKType.h
  53. +13 −0 src/PKVariableSymbol.h
  54. +13 −0 src/PKVariableSymbol.m
  55. +0 −56 src/TDGrammarParser.h
  56. +0 −640 src/TDGrammarParser.m
  57. +2 −2 test/EBNFParser.m
  58. +2 −1 test/EBNFParserTest.m
  59. +1 −1 test/ERBTest.m
  60. +39 −11 test/PKAST.m
  61. +0 −17 test/PKNodeBase.h
  62. +0 −17 test/PKNodeBase.m
  63. +0 −14 test/PKNodeCardinal.h
  64. +0 −22 test/PKNodeCardinal.m
  65. +0 −13 test/PKNodeCollection.h
  66. +0 −22 test/PKNodeCollection.m
  67. +0 −13 test/PKNodeComposite.h
  68. +0 −22 test/PKNodeComposite.m
  69. +0 −13 test/PKNodeConstant.h
  70. +0 −22 test/PKNodeConstant.m
  71. +0 −29 test/PKNodeDelimited.m
  72. +0 −22 test/PKNodeMultiple.m
  73. +0 −28 test/PKNodePattern.m
  74. +0 −30 test/PKNodeTypes.h
  75. +0 −28 test/PKNodeVariable.m
  76. +0 −49 test/PKNodeVisitor.h
  77. +0 −363 test/PKNodeVisitor.m
  78. +3 −1 test/SAXTest.m
  79. +1 −1 test/SRGSParserTest.m
  80. +59 −59 test/TDDelimitStateTest.m
  81. +1 −1 test/TDJsonParserTest.m
  82. +2 −2 test/TDNSPredicateEvaluatorTest.m
  83. +43 −0 test/TDParseTreeTest.m
  84. +0 −33 test/TDParserFactory.h
  85. +0 −688 test/TDParserFactory.m
  86. +751 −33 test/TDParserFactoryASTTest.m
  87. +144 −35 test/TDParserFactoryParserTest.m
  88. +18 −18 test/TDParserFactoryPatternTest.m
  89. +13 −0 test/TDParserFactorySymbolTableTest.h
  90. +687 −0 test/TDParserFactorySymbolTableTest.m
  91. +675 −656 test/TDParserFactoryTest.m
  92. +6 −6 test/TDParserFactoryTest2.m
  93. +38 −31 test/TDParserFactoryTest3.m
  94. +16 −16 test/TDRegexAssembler.m
  95. +7 −1 test/TDRegexMatcher.m
  96. +5 −1 test/TDRegexMatcherTest.m
  97. +4 −4 test/TDRepetitionTest.m
  98. +1 −1 test/TDTestScaffold.m
  99. +3 −3 test/TDTokenizerBlocksTest.m
  100. +59 −1 test/TDTokenizerTest.m
  101. +18 −7 test/TDXMLParserTest.m
  102. +3 −3 test/TDXmlText.m
  103. +4 −3 test/TDXmlTokenizerTest.m
  104. +1 −1 test/XMLReaderTest.m
  105. +1 −1 test/XPathParser.m
  106. +1 −1 test/XPathParserGrammarTest.m
  107. +2 −2 test/XPathParserTest.m
  108. +13 −0 test/nodes/PKAlternationNode.h
  109. +28 −0 test/nodes/PKAlternationNode.m
  110. +42 −0 test/nodes/PKBaseNode.h
  111. +81 −0 test/nodes/PKBaseNode.m
  112. +14 −0 test/nodes/PKCardinalNode.h
  113. +55 −0 test/nodes/PKCardinalNode.m
  114. +13 −0 test/nodes/PKCollectionNode.h
  115. +47 −0 test/nodes/PKCollectionNode.m
  116. +13 −0 test/nodes/PKCompositeNode.h
  117. +44 −0 test/nodes/PKCompositeNode.m
  118. +14 −0 test/nodes/PKConstantNode.h
  119. +75 −0 test/nodes/PKConstantNode.m
  120. +3 −3 test/{PKNodeVariable.h → nodes/PKDefinitionNode.h}
  121. +75 −0 test/nodes/PKDefinitionNode.m
  122. +2 −2 test/{PKNodeDelimited.h → nodes/PKDelimitedNode.h}
  123. +72 −0 test/nodes/PKDelimitedNode.m
  124. +14 −0 test/nodes/PKLiteralNode.h
  125. +33 −0 test/nodes/PKLiteralNode.m
  126. +2 −2 test/{PKNodeMultiple.h → nodes/PKMultipleNode.h}
  127. +28 −0 test/nodes/PKMultipleNode.m
  128. +42 −0 test/nodes/PKNodeVisitor.h
  129. +2 −2 test/{PKNodeOptional.h → nodes/PKOptionalNode.h}
  130. +4 −4 test/{PKNodeOptional.m → nodes/PKOptionalNode.m}
  131. +2 −2 test/{PKNodePattern.h → nodes/PKPatternNode.h}
  132. +75 −0 test/nodes/PKPatternNode.m
  133. +13 −0 test/nodes/PKReferenceNode.h
  134. +41 −0 test/nodes/PKReferenceNode.m
  135. +13 −0 test/nodes/PKRootNode.h
  136. +22 −0 test/nodes/PKRootNode.m
View
@@ -16,12 +16,10 @@
<string>${PRODUCT_NAME}</string>
<key>CFBundlePackageType</key>
<string>FMWK</string>
- <key>CFBundleShortVersionString</key>
- <string>0.5</string>
<key>CFBundleSignature</key>
<string>????</string>
<key>CFBundleVersion</key>
- <string>0.5</string>
+ <string>0.6</string>
<key>NSPrincipalClass</key>
<string></string>
</dict>
View
@@ -12,11 +12,9 @@
<string>6.0</string>
<key>CFBundlePackageType</key>
<string>FMWK</string>
- <key>CFBundleShortVersionString</key>
- <string>0.5</string>
<key>CFBundleSignature</key>
<string>????</string>
<key>CFBundleVersion</key>
- <string>0.5</string>
+ <string>0.6</string>
</dict>
</plist>

Large diffs are not rendered by default.

Oops, something went wrong.
@@ -399,7 +399,7 @@ - (void)doFactory {
//// TDEqualObjects(@"[bar, bat]bar/bat^", [res description]);
//// [mock verify];
-// NSString *g = @"@delimitState = '$'; @delimitedString = '$' '%' nil; @start = DelimitedString('$', '%');";
+// NSString *g = @"@delimitState = '$'; @delimitedString = '$' '%' nil; @start = %{'$', '%'};";
// PKParser *lp = [[PKParserFactory factory] parserFromGrammar:g assembler:nil];
//
// NSString *s = @"$foo%";
@@ -485,33 +485,54 @@ - (void)doTestGrammar {
path = [@"~/Desktop/input.txt" stringByExpandingTildeInPath];
NSString *s = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:nil];
- PKAssembly *res = [p parse:s error:nil];
-// p.tokenizer.string = s;
-// PKAssembly *res = [p bestMatchFor:[PKTokenAssembly assemblyWithTokenizer:p.tokenizer]];
+// PKAssembly *res = [p parse:s error:nil];
+ p.tokenizer.string = s;
+ PKAssembly *res = [p bestMatchFor:[PKTokenAssembly assemblyWithTokenizer:p.tokenizer]];
NSLog(@"p %@", p);
NSLog(@"res %@", res);
}
+
+- (void)doTestSqliteGrammar {
+
+ // NSString *path = [[NSBundle mainBundle] pathForResource:@"date" ofType:@"grammar"];
+ NSString *path = [@"~/work/parsekit/trunk/res/sqlite.grammar" stringByExpandingTildeInPath];
+ NSString *g = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:nil];
+ PKParser *p = [[PKParserFactory factory] parserFromGrammar:g assembler:self error:nil];
+
+ path = [@"~/work/parsekit/trunk/res/sqlite_input.txt" stringByExpandingTildeInPath];
+ NSString *s = [NSString stringWithContentsOfFile:path encoding:NSUTF8StringEncoding error:nil];
+
+ // PKAssembly *res = [p parse:s error:nil];
+ p.tokenizer.string = s;
+ PKAssembly *res = [p bestMatchFor:[PKTokenAssembly assemblyWithTokenizer:p.tokenizer]];
+ NSLog(@"p %@", p);
+ NSLog(@"res %@", res);
+
+}
+
+
- (void)parser:(PKParser *)p didMatchTag:(PKAssembly *)a {
NSLog(@"%s", __PRETTY_FUNCTION__);
NSLog(@"%@", a);
}
-- (void)parser:(PKParser *)p didMatchExpr:(PKAssembly *)a {
- NSLog(@"%s %@", __PRETTY_FUNCTION__, a);
- NSArray *toks = [a objectsAbove:nil];
-
- double total = 0.0;
- for (PKToken *tok in toks) {
- double n = tok.floatValue;
- total += n;
- }
-
- a.target = [NSNumber numberWithDouble:total];
-}
+//- (void)parser:(PKParser *)p didMatchExpr:(PKAssembly *)a {
+// NSLog(@"%s %@", __PRETTY_FUNCTION__, a);
+// NSArray *toks = [a objectsAbove:nil];
+//
+// double total = 0.0;
+// for (PKToken *tok in toks) {
+// double n = tok.floatValue;
+// total += n;
+// }
+//
+// a.target = [NSNumber numberWithDouble:total];
+//}
+
//- (void)parser:(PKParser *)p didMatchTerm:(PKAssembly *)a {
// NSLog(@"%s %@", __PRETTY_FUNCTION__, a);
@@ -537,30 +558,28 @@ - (void)parser:(PKParser *)p didMatchTree:(PKAssembly *)a {
}
+- (void)parser:(PKParser *)p didMatchFunctionKeyword:(PKAssembly *)a {
+ [a push:[NSNull null]];
+}
- - (void)parser:(PKParser *)p didMatchFunctionKeyword:(PKAssembly *)a {
- [a push:[NSNull null]];
- }
-
-
- - (void)parser:(PKParser *)p didMatchFunctionCloseCurly:(PKAssembly *)a {
- id obj = [a pop];
- NSAssert(obj == [NSNull null], @"null should be on the top of the stack");
- }
+- (void)parser:(PKParser *)p didMatchFunctionCloseCurly:(PKAssembly *)a {
+ id obj = [a pop];
+ NSAssert(obj == [NSNull null], @"null should be on the top of the stack");
+}
- - (void)parser:(PKParser *)p didMatchVarDecl:(PKAssembly *)a {
- id obj = [a pop];
- if (obj == [NSNull null]) {
- [a push:obj]; // we're in a function. put the null back and bail.
- } else {
- PKToken *fence = [PKToken tokenWithTokenType:PKTokenTypeWord stringValue:@"var" floatValue:0.0];
- NSArray *toks = [a objectsAbove:fence]; // get all the tokens for the var decl
- // now do whatever you want with the var decl tokens here.
- }
-
+- (void)parser:(PKParser *)p didMatchVarDecl:(PKAssembly *)a {
+ id obj = [a pop];
+ if (obj == [NSNull null]) {
+ [a push:obj]; // we're in a function. put the null back and bail.
+ } else {
+ PKToken *fence = [PKToken tokenWithTokenType:PKTokenTypeWord stringValue:@"var" floatValue:0.0];
+ NSArray *toks = [a objectsAbove:fence]; // get all the tokens for the var decl
+ // now do whatever you want with the var decl tokens here.
}
+
+}
@@ -569,6 +588,7 @@ - (IBAction)run:(id)sender {
[self doTestGrammar];
+ //[self doTestSqliteGrammar];
// [self doPlistParser];
// [self doHtmlSyntaxHighlighter];
@@ -577,7 +597,7 @@ - (IBAction)run:(id)sender {
// [self doJSParser];
- // [self doProf];
+// [self doProf];
//[self doJavaScriptGrammarParser];
@@ -1,29 +1,29 @@
<?xml version="1.0" encoding="UTF-8"?>
<archive type="com.apple.InterfaceBuilder3.Cocoa.XIB" version="7.10">
<data>
- <int key="IBDocument.SystemTarget">1070</int>
- <string key="IBDocument.SystemVersion">11E53</string>
- <string key="IBDocument.InterfaceBuilderVersion">2182</string>
- <string key="IBDocument.AppKitVersion">1138.47</string>
- <string key="IBDocument.HIToolboxVersion">569.00</string>
+ <int key="IBDocument.SystemTarget">1080</int>
+ <string key="IBDocument.SystemVersion">12C2034</string>
+ <string key="IBDocument.InterfaceBuilderVersion">3084</string>
+ <string key="IBDocument.AppKitVersion">1187.34</string>
+ <string key="IBDocument.HIToolboxVersion">625.00</string>
<object class="NSMutableDictionary" key="IBDocument.PluginVersions">
<string key="NS.key.0">com.apple.InterfaceBuilder.CocoaPlugin</string>
- <string key="NS.object.0">2182</string>
+ <string key="NS.object.0">3084</string>
</object>
<object class="NSArray" key="IBDocument.IntegratedClassDependencies">
<bool key="EncodedWithXMLCoder">YES</bool>
- <string>NSScroller</string>
- <string>NSMenuItem</string>
- <string>NSMenu</string>
- <string>NSTextFieldCell</string>
- <string>NSScrollView</string>
- <string>NSButtonCell</string>
<string>NSButton</string>
+ <string>NSButtonCell</string>
<string>NSCustomObject</string>
+ <string>NSMenu</string>
+ <string>NSMenuItem</string>
+ <string>NSScrollView</string>
+ <string>NSScroller</string>
+ <string>NSTextField</string>
+ <string>NSTextFieldCell</string>
<string>NSTextView</string>
<string>NSView</string>
<string>NSWindowTemplate</string>
- <string>NSTextField</string>
</object>
<object class="NSArray" key="IBDocument.PluginDependencies">
<bool key="EncodedWithXMLCoder">YES</bool>
@@ -1115,7 +1115,7 @@
<reference key="NSNextKeyView" ref="525752343"/>
<bool key="NSEnabled">YES</bool>
<object class="NSButtonCell" key="NSCell" id="449794129">
- <int key="NSCellFlags">67239424</int>
+ <int key="NSCellFlags">67108864</int>
<int key="NSCellFlags2">134217728</int>
<string key="NSContents">Run</string>
<object class="NSFont" key="NSSupport">
@@ -1124,13 +1124,14 @@
<int key="NSfFlags">1044</int>
</object>
<reference key="NSControlView" ref="663773455"/>
- <int key="NSButtonFlags">-2038284033</int>
+ <int key="NSButtonFlags">-2038284288</int>
<int key="NSButtonFlags2">129</int>
<string key="NSAlternateContents"/>
<string type="base64-UTF8" key="NSKeyEquivalent">DQ</string>
<int key="NSPeriodicDelay">200</int>
<int key="NSPeriodicInterval">25</int>
</object>
+ <bool key="NSAllowsLogicalLayoutDirection">NO</bool>
</object>
<object class="NSTextField" id="772280458">
<reference key="NSNextResponder" ref="127129819"/>
@@ -1141,7 +1142,7 @@
<reference key="NSNextKeyView" ref="663773455"/>
<bool key="NSEnabled">YES</bool>
<object class="NSTextFieldCell" key="NSCell" id="728180371">
- <int key="NSCellFlags">67239424</int>
+ <int key="NSCellFlags">67108864</int>
<int key="NSCellFlags2">272629760</int>
<string key="NSContents">The purpose of this application is only to run arbtrary test code in "Debug" mode using GDB and breakpoints (which does not seem to be possible fom the UnitTest Bundle). To use, copy some unit test code over to the -[DebugAppDelegate run:] action method which is executed when the "Run" button below is clicked.</string>
<object class="NSFont" key="NSSupport">
@@ -1169,6 +1170,7 @@
</object>
</object>
</object>
+ <bool key="NSAllowsLogicalLayoutDirection">NO</bool>
</object>
<object class="NSScrollView" id="525752343">
<reference key="NSNextResponder" ref="127129819"/>
@@ -1183,10 +1185,10 @@
<object class="NSTextView" id="432740214">
<reference key="NSNextResponder" ref="1018750903"/>
<int key="NSvFlags">2322</int>
- <string key="NSFrameSize">{462, 84}</string>
+ <string key="NSFrameSize">{462, 259}</string>
<reference key="NSSuperview" ref="1018750903"/>
<reference key="NSWindow"/>
- <reference key="NSNextKeyView" ref="1066186764"/>
+ <reference key="NSNextKeyView" ref="1016141326"/>
<object class="NSTextContainer" key="NSTextContainer" id="152014479">
<object class="NSLayoutManager" key="NSLayoutManager">
<object class="NSTextStorage" key="NSTextStorage">
@@ -1377,15 +1379,15 @@
<bool key="EncodedWithXMLCoder">YES</bool>
<reference ref="152014479"/>
</object>
- <int key="NSLMFlags">6</int>
+ <int key="NSLMFlags">38</int>
<nil key="NSDelegate"/>
</object>
<reference key="NSTextView" ref="432740214"/>
<double key="NSWidth">462</double>
<int key="NSTCFlags">1</int>
</object>
<object class="NSTextViewSharedData" key="NSSharedData">
- <int key="NSFlags">11621</int>
+ <int key="NSFlags">67120485</int>
<int key="NSTextCheckingTypes">0</int>
<nil key="NSMarkedAttributes"/>
<object class="NSColor" key="NSBackgroundColor">
@@ -1438,7 +1440,6 @@
</object>
<int key="NSTVFlags">6</int>
<string key="NSMaxSize">{3081, 10000000}</string>
- <string key="NSMinize">{223, 0}</string>
<nil key="NSDelegate"/>
</object>
</object>
@@ -1462,9 +1463,7 @@
<integer value="5"/>
<object class="NSURL">
<nil key="NS.base"/>
- <object class="NSMutableString" key="NS.relative">
- <characters key="NS.bytes">file://localhost/Applications/Xcode.app/Contents/SharedFrameworks/DVTKit.framework/Resources/DVTIbeamCursor.tiff</characters>
- </object>
+ <string key="NS.relative">file://localhost/Applications/Xcode.app/Contents/SharedFrameworks/DVTKit.framework/Resources/DVTIbeamCursor.tiff</string>
</object>
</object>
</object>
@@ -1479,9 +1478,10 @@
<object class="NSScroller" id="1066186764">
<reference key="NSNextResponder" ref="525752343"/>
<int key="NSvFlags">256</int>
- <string key="NSFrame">{{448, 1}, {15, 259}}</string>
+ <string key="NSFrame">{{447, 1}, {16, 259}}</string>
<reference key="NSSuperview" ref="525752343"/>
<reference key="NSWindow"/>
+ <bool key="NSAllowsLogicalLayoutDirection">NO</bool>
<reference key="NSTarget" ref="525752343"/>
<string key="NSAction">_doScroller:</string>
<double key="NSPercent">0.984375</double>
@@ -1493,6 +1493,7 @@
<reference key="NSSuperview" ref="525752343"/>
<reference key="NSWindow"/>
<reference key="NSNextKeyView" ref="1018750903"/>
+ <bool key="NSAllowsLogicalLayoutDirection">NO</bool>
<int key="NSsFlags">1</int>
<reference key="NSTarget" ref="525752343"/>
<string key="NSAction">_doScroller:</string>
@@ -1503,11 +1504,14 @@
<string key="NSFrame">{{20, 20}, {464, 261}}</string>
<reference key="NSSuperview" ref="127129819"/>
<reference key="NSWindow"/>
- <reference key="NSNextKeyView" ref="1016141326"/>
+ <reference key="NSNextKeyView" ref="1018750903"/>
<int key="NSsFlags">133138</int>
<reference key="NSVScroller" ref="1066186764"/>
<reference key="NSHScroller" ref="1016141326"/>
<reference key="NSContentView" ref="1018750903"/>
+ <double key="NSMinMagnification">0.25</double>
+ <double key="NSMaxMagnification">4</double>
+ <double key="NSMagnification">1</double>
</object>
</object>
<string key="NSFrameSize">{504, 447}</string>
@@ -2040,6 +2044,14 @@
<int key="connectionID">422</int>
</object>
<object class="IBConnectionRecord">
+ <object class="IBOutletConnection" key="connection">
+ <string key="label">initialFirstResponder</string>
+ <reference key="source" ref="100995220"/>
+ <reference key="destination" ref="663773455"/>
+ </object>
+ <int key="connectionID">463</int>
+ </object>
+ <object class="IBConnectionRecord">
<object class="IBActionConnection" key="connection">
<string key="label">run:</string>
<reference key="source" ref="1006034491"/>
@@ -3327,7 +3339,7 @@
<reference key="dict.values" ref="1049"/>
</object>
<nil key="sourceID"/>
- <int key="maxID">462</int>
+ <int key="maxID">463</int>
</object>
<object class="IBClassDescriber" key="IBDocument.Classes">
<object class="NSMutableArray" key="referencedPartialClassDescriptions">
Oops, something went wrong.

0 comments on commit e7e1984

Please sign in to comment.