Skip to content

Commit

Permalink
Text parser system: experimental support for JSON + displaying a "unk…
Browse files Browse the repository at this point in the history 
…nown/ok/broken" document integrity icon in the location bar
  • Loading branch information
@rsms
rsms committed Feb 1, 2011
1 parent cb7eb5b commit 77a1c6c
Show file tree
Hide file tree
Showing 20 changed files with 385 additions and 157 deletions.
46 changes: 38 additions & 8 deletions kod.xcodeproj/project.pbxproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
/* End PBXAggregateTarget section */

/* Begin PBXBuildFile section */
3A1007CD12F81838003CA014 /* KParseStatusDecoration.mm in Sources */ = {isa = PBXBuildFile; fileRef = 3A1007CC12F81838003CA014 /* KParseStatusDecoration.mm */; };
3A1007DC12F819D6003CA014 /* parse-status-broken.png in Resources */ = {isa = PBXBuildFile; fileRef = 3A1007DA12F819D6003CA014 /* parse-status-broken.png */; };
3A1007DD12F819D6003CA014 /* parse-status-ok.png in Resources */ = {isa = PBXBuildFile; fileRef = 3A1007DB12F819D6003CA014 /* parse-status-ok.png */; };
3A158ADB128A09D300339E6E /* ICUMatcher.m in Sources */ = {isa = PBXBuildFile; fileRef = 3A158AD5128A09D300339E6E /* ICUMatcher.m */; };
3A158ADC128A09D300339E6E /* ICUPattern.m in Sources */ = {isa = PBXBuildFile; fileRef = 3A158AD7128A09D300339E6E /* ICUPattern.m */; };
3A158ADE128A09D300339E6E /* NSStringICUAdditions.m in Sources */ = {isa = PBXBuildFile; fileRef = 3A158ADA128A09D300339E6E /* NSStringICUAdditions.m */; };
Expand Down Expand Up @@ -181,14 +184,14 @@
isa = PBXContainerItemProxy;
containerPortal = 3A928BEF1227143800DA7E43 /* Project object */;
proxyType = 1;
remoteGlobalIDString = 3A8D377E12F6D0CA007666FC /* lua */;
remoteGlobalIDString = 3A8D377E12F6D0CA007666FC;
remoteInfo = lua;
};
3A8D37C512F6DE8F007666FC /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 3A928BEF1227143800DA7E43 /* Project object */;
proxyType = 1;
remoteGlobalIDString = 3A8D377012F6CA88007666FC /* gazelle */;
remoteGlobalIDString = 3A8D377012F6CA88007666FC;
remoteInfo = gazelle;
};
3AAA9FB612BFC8900079F079 /* PBXContainerItemProxy */ = {
Expand Down Expand Up @@ -281,6 +284,10 @@
/* End PBXCopyFilesBuildPhase section */

/* Begin PBXFileReference section */
3A1007CB12F81838003CA014 /* KParseStatusDecoration.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = KParseStatusDecoration.h; sourceTree = "<group>"; };
3A1007CC12F81838003CA014 /* KParseStatusDecoration.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = KParseStatusDecoration.mm; sourceTree = "<group>"; };
3A1007DA12F819D6003CA014 /* parse-status-broken.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "parse-status-broken.png"; sourceTree = "<group>"; };
3A1007DB12F819D6003CA014 /* parse-status-ok.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "parse-status-ok.png"; sourceTree = "<group>"; };
3A158AD4128A09D300339E6E /* ICUMatcher.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ICUMatcher.h; sourceTree = "<group>"; };
3A158AD5128A09D300339E6E /* ICUMatcher.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = ICUMatcher.m; sourceTree = "<group>"; };
3A158AD6128A09D300339E6E /* ICUPattern.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ICUPattern.h; sourceTree = "<group>"; };
Expand Down Expand Up @@ -574,6 +581,28 @@
/* End PBXFrameworksBuildPhase section */

/* Begin PBXGroup section */
3A1007CE12F8199E003CA014 /* XIBs */ = {
isa = PBXGroup;
children = (
3A928C1B122714EE00DA7E43 /* MainMenu.xib */,
3A7E71121286EAE800D85E4C /* BrowserWindow.xib */,
3A9291171229B93C00DA7E43 /* Toolbar.xib */,
3A510B3B12C26EFE00AAC1D5 /* terminal-usage.xib */,
);
name = XIBs;
sourceTree = "<group>";
};
3A1007D012F819C1003CA014 /* gfx */ = {
isa = PBXGroup;
children = (
3A1007DA12F819D6003CA014 /* parse-status-broken.png */,
3A1007DB12F819D6003CA014 /* parse-status-ok.png */,
3AAC03EE12BD2FC2001BDC39 /* sidebar-closed.png */,
3AAC03EF12BD2FC2001BDC39 /* sidebar-open.png */,
);
name = gfx;
sourceTree = "<group>";
};
3A158AD3128A09D300339E6E /* ICU */ = {
isa = PBXGroup;
children = (
Expand Down Expand Up @@ -834,15 +863,11 @@
3AA8142612C390CE0085F086 /* about.md */,
3AA818D312C4DD330085F086 /* changelog */,
3AA817F812C4A6CC0085F086 /* fonts */,
3AAC03EE12BD2FC2001BDC39 /* sidebar-closed.png */,
3AAC03EF12BD2FC2001BDC39 /* sidebar-open.png */,
3A1007D012F819C1003CA014 /* gfx */,
3A158DF1128A282F00339E6E /* style */,
3A928C1A122714EE00DA7E43 /* Info.plist */,
3A40F92E12AFF21A007B5879 /* main.js */,
3A7E71121286EAE800D85E4C /* BrowserWindow.xib */,
3A928C1B122714EE00DA7E43 /* MainMenu.xib */,
3A9291171229B93C00DA7E43 /* Toolbar.xib */,
3A510B3B12C26EFE00AAC1D5 /* terminal-usage.xib */,
3A1007CE12F8199E003CA014 /* XIBs */,
);
path = resources;
sourceTree = "<group>";
Expand Down Expand Up @@ -958,6 +983,8 @@
3A62BA48129D756100822274 /* KAutocompleteTextFieldEditor.mm */,
3A62B9E4129D652700822274 /* KTextFieldDecoration.h */,
3A62B9E5129D652700822274 /* KTextFieldDecoration.mm */,
3A1007CB12F81838003CA014 /* KParseStatusDecoration.h */,
3A1007CC12F81838003CA014 /* KParseStatusDecoration.mm */,
3A811A7B12AE813800BC08AE /* KModeTextFieldDecoration.h */,
3A811A7C12AE813800BC08AE /* KModeTextFieldDecoration.mm */,
);
Expand Down Expand Up @@ -1273,6 +1300,8 @@
3AA817FF12C4A6CC0085F086 /* fonts in Resources */,
3AA818D412C4DD330085F086 /* changelog in Resources */,
3A37B39412DA2A4200ED811C /* ast-viewer.xib in Resources */,
3A1007DC12F819D6003CA014 /* parse-status-broken.png in Resources */,
3A1007DD12F819D6003CA014 /* parse-status-ok.png in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down Expand Up @@ -1443,6 +1472,7 @@
3A8D37DA12F6DF92007666FC /* ASTNode.mm in Sources */,
3A8D37DB12F6DF92007666FC /* ASTParser.mm in Sources */,
3A8D383212F6E995007666FC /* AST.mm in Sources */,
3A1007CD12F81838003CA014 /* KParseStatusDecoration.mm in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
Expand Down
Binary file added resources/parse-status-broken.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added resources/parse-status-ok.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added resources/parse-status.psd
View file
Binary file not shown.
25 changes: 15 additions & 10 deletions src/AST.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,12 @@
#include <string>
#include <tr1/memory>

typedef enum {
KParseStatusUnknown = 0,
KParseStatusOK,
KParseStatusBroken,
} KParseStatus;

@class KDocument;

namespace kod {
Expand All @@ -23,24 +29,23 @@ class AST {
explicit AST(KDocument *document=NULL);
~AST() {}

const ASTNodePtr &rootNode() const { return rootNode_; }
bool setRootNode(ASTNodePtr rootNode) {
// TODO(rsms): using CAS or spinlock
h_atomic_barrier();
rootNode_ = rootNode;
}
ASTNodePtr &rootNode() const { return parser_->rootNode(); }

bool parse();
bool parseEdit(NSUInteger changeLocation, long changeDelta);

KParseStatus status();

bool isOpenEnded() {
return parser_->currentNode().get() != parser_->rootNode().get();
}

protected:
KDocument *document_; // weak, owns us
ASTParserPtr parser_;
GrammarPtr grammar_;
ASTNodePtr rootNode_;

// state
bool isOpenEnded_;
bool needFullParse_;
gzl_status status_;
};

typedef std::tr1::shared_ptr<AST> ASTPtr;
Expand Down
104 changes: 84 additions & 20 deletions src/AST.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,11 @@

AST::AST(KDocument *document)
: document_(document)
, isOpenEnded_(false) {
, status_(GZL_STATUS_OK)
, needFullParse_(true) {
// xxx fixme
grammar_.reset(new Grammar);
parser_.reset(new ASTParser);
grammar_.reset(new Grammar("JSON"));
parser_.reset(new ASTParser());
if (grammar_->loadFile("/Users/rasmus/src/gazelle/test2/json.gzc")) {
parser_->setGrammar(grammar_.get());
} else {
Expand All @@ -23,35 +24,98 @@


bool AST::parse() {

Grammar *grammar = new Grammar;
assert(grammar->loadFile("/Users/rasmus/src/gazelle/test2/json.gzc"));
ASTParser parser;
parser.setGrammar(grammar);
parser_->reset();
parser_->setGrammar(grammar_.get());

NSString *text = [[document_ textView] textStorage].string;
const char *source = [text UTF8String]; // FIXME unichar
//const char *source = "{\n\"foo\": 12.34, \"bar\": 4\n}";

parser.setSource(source);
parser_->setSource(source);
DLOG("parsing");
gzl_status status = parser.parse(source, 0, true);

DLOG("parse status: %s", ASTParser::gazelleStatusString(status));
status_ = parser_->parse(source, 0, true);

isOpenEnded_ =
parser.currentASTNode().get() != parser.rootASTNode().get();
DLOG("parse status: %s", ASTParser::gazelleStatusString(status_));
DLOG("isOpenEnded: %d", isOpenEnded());
//DLOG("AST:\n%s", parser_->rootNode()->inspect().c_str());

fprintf(stderr, "isOpenEnded_: %d\n", isOpenEnded_);
fprintf(stderr, "AST:\n%s\n", parser.rootASTNode()->inspect().c_str());
needFullParse_ = false;

return status == GZL_STATUS_OK;
[document_ ASTWasUpdated];
return true;
}


bool AST::parseEdit(NSUInteger changeLocation, long changeDelta) {
// if a full parse is needed, take the "quick" route
if (needFullParse_)
return parse();

// find affected node
NSRange mrange =
ASTParser::absoluteSourceRangeFromChangeInfo(changeLocation, changeDelta);
DLOG("mrange = %@", NSStringFromRange(mrange));
NSUInteger continueAtSourceLocation = 0;
size_t affectedParentOffset = 0;
ASTNode *continueAtNode = NULL;
ASTNode *affectedNode =
parser_->rootNode()->findAffectedBranch(mrange,
0,
&continueAtSourceLocation,
&continueAtNode,
&affectedParentOffset);
if (!continueAtNode)
return parse(); // FIXME
// TODO: logic instead of assertions
kassert(continueAtNode != NULL);
kassert(affectedNode != NULL);
DLOG("affected node: %s", affectedNode->inspect(false).c_str());
DLOG("continue after node: %s", continueAtNode->inspect(false).c_str());

// tell parser to replace the old node with the new one
parser_->setExplicitNextChildNodeIndex(affectedParentOffset);

// resuscitate parser state
gzl_parse_state *newState = continueAtNode->parseState();
assert(newState != NULL);
parser_->setState(gzl_dup_parse_state(newState));

// get source
NSString *text = [[document_ textView] textStorage].string;
//parser.setSource([text UTF8String]);
//parser.setSource(source);
//gzl_status status = parser.parse(source, 0, true);
const char *source = [text UTF8String]; // FIXME unichar

// parse
size_t sourceLen = strlen(source);
//DLOG("source -> (%zu) '%s'", sourceLen, source);
parser_->setSource(source);
parser_->currentNode() = affectedNode->parentNode();
DLOG("currentNode(): %s", parser_->currentNode()->inspect(false).c_str());
status_ = parser_->parse(source, sourceLen, true);

// check status
DLOG("parse status: %s", ASTParser::gazelleStatusString(status_));
DLOG("isOpenEnded: %d", isOpenEnded());
//DLOG("AST:\n%s\n", parser_->rootNode()->inspect().c_str());

[document_ ASTWasUpdated];
return true;
}


KParseStatus AST::status() {
switch (status_) {
case GZL_STATUS_OK:
case GZL_STATUS_HARD_EOF:
return KParseStatusOK;
case GZL_STATUS_ERROR:
case GZL_STATUS_PREMATURE_EOF_ERROR:
case GZL_STATUS_CANCELLED:
return KParseStatusBroken;
default:
return KParseStatusUnknown;
}
}




5 changes: 3 additions & 2 deletions src/ASTNode.mm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@


ASTNode::~ASTNode() {
fprintf(stderr, "ASTNode '%s' DEALLOC\n", ruleName_);
if (parserState_) {
gzl_free_parse_state(parserState_);
parserState_ = NULL;
Expand Down Expand Up @@ -130,11 +131,11 @@
depth*2, "",
ruleName(), sourceRange().location, sourceRange().length);
str.append(buf);
if (parserState_) {
/*if (parserState_) {
snprintf(buf, sizeof(buf)-1, ", snapshot: { offset:%zu, stackSize: %d } ",
parserState_->offset.byte, parserState_->parse_stack_len);
str.append(buf);
}
}*/
if (!childNodes().empty()) {
if (!deep) {
snprintf(buf, sizeof(buf)-1, ", childNodes: (%zu) },",
Expand Down
37 changes: 12 additions & 25 deletions src/ASTParser.hh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,23 +14,19 @@ class ASTParser : public gazelle::Parser {
size_t sourceLen_;

// Current AST node
ASTNodePtr currentASTNode_;
ASTNodePtr rootASTNode_;
ASTNodePtr currentNode_;
ASTNodePtr rootNode_;

NSUInteger explicitNextChildNodeIndex_;

public:

ASTParser() : explicitNextChildNodeIndex_(NSNotFound) { reset(); }
/*ASTParser(gazelle::Grammar *grammar) : explicitNextChildNodeIndex_(NSNotFound) {
reset();
}*/
explicit ASTParser() : explicitNextChildNodeIndex_(NSNotFound) { reset(); }
virtual ~ASTParser() {}

void reset() {
rootASTNode_.reset(new ASTNode("root", 0));
currentASTNode_ = rootASTNode_;
rootNode_.reset(new ASTNode("root", 0));
currentNode_ = rootNode_;
setState(gzl_alloc_parse_state());
}

Expand All @@ -43,29 +39,20 @@ class ASTParser : public gazelle::Parser {
explicitNextChildNodeIndex_ = nextChildNodeIndex;
}

void pushASTNode(ASTNode *node) {
ASTNodePtr nodeptr(node);
node->parentNode() = currentASTNode_;
if (explicitNextChildNodeIndex_ != NSNotFound) {
currentASTNode_->childNodes()[explicitNextChildNodeIndex_] = nodeptr;
explicitNextChildNodeIndex_ = NSNotFound;
} else {
currentASTNode_->childNodes().push_back(nodeptr);
}
currentASTNode_ = nodeptr;
}
void pushASTNode(ASTNode *node);

void popASTNode() {
//assert(currentASTNode_->parentNode().get());
currentASTNode_ = currentASTNode_->parentNode();
//assert(currentNode_->parentNode().get());
currentNode_ = currentNode_->parentNode();
}

ASTNodePtr &currentASTNode() { return currentASTNode_; }
ASTNodePtr &rootASTNode() { return rootASTNode_; }
ASTNodePtr &currentNode() { return currentNode_; }
ASTNodePtr &rootNode() { return rootNode_; }

void onWillStartRule(gzl_rtn *frame, const char *name, gzl_offset *offset);
void onDidStartRule(gzl_rtn_frame *frame, const char *name);
void onEndRule(gzl_rtn_frame *frame, const char *name);
void onWillEndRule(gzl_rtn_frame *frame, const char *name);
void onDidEndRule(gzl_rtn_frame *frame, const char *name);
void onTerminal(gzl_terminal *terminal);
void onUnknownTransitionError(int ch);
void onUnexpectedTerminalError(gzl_terminal *terminal);
Expand Down
Loading

0 comments on commit 77a1c6c

Please sign in to comment.