Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

*MASSIVE* speedups.

  • Loading branch information...
commit 05bdec4e6f201cd843753862c862f984d700d076 1 parent 78b2368
@bl0b authored
Showing with 33 additions and 14 deletions.
  1. +4 −3 jupyLR/automaton.py
  2. +13 −4 jupyLR/lr.py
  3. +16 −7 jupyLR/parser.py
View
7 jupyLR/automaton.py
@@ -67,9 +67,6 @@ def __recognize(self, token_stream):
S.count_active = 1
prev_tok = INITIAL_TOKEN
for cur_tok in token_stream:
- if self.debug:
- print "On token", cur_tok
- S.dump()
if len(S.active) == 0:
if not self.error_detected(prev_tok, S.previously_active):
break
@@ -103,6 +100,10 @@ def __recognize(self, token_stream):
#print "pre merge", S.active
# Merge states
S.merge()
+ # A little bit of feedback
+ if self.debug:
+ print "On token", cur_tok
+ S.dump()
return None
def __call__(self, text):
View
17 jupyLR/lr.py
@@ -11,7 +11,10 @@ def expand_itemset(itemset, R):
def expand_itemset2(itemset, R):
- return imap(lambda x: x[:1] + expand_item(x, R), itemset)
+ for item in itemset:
+ rule = R[item[0]]
+ yield item[0], rule[1], item[1], rule[0]
+ #return imap(lambda x: x[:1] + expand_item(x, R), itemset)
def itemstr(item, R):
@@ -59,15 +62,21 @@ def closure(itemset, R):
"the epsilon-closure of this item set"
C = set(itemset)
last = -1
+ visited = set()
while len(C) != last:
last = len(C)
Ctmp = set()
for item in C:
- elems, i, name = expand_item(item, R)
+ r, i = item
+ name, elems, commit = R[r]
+ #elems, i, name = expand_item(item, R)
if i == len(elems):
continue
- if elems[i] in R:
- Ctmp.update((r, 0) for r in R[elems[i]])
+ if elems[i] in R and elems[i] not in visited:
+ visited.add(elems[i])
+ for r in R[elems[i]]:
+ Ctmp.add((r, 0))
+ #Ctmp.update((r, 0) for r in R[elems[i]])
C.update(Ctmp)
return C
View
23 jupyLR/parser.py
@@ -144,11 +144,10 @@ class parser(object):
def __init__(self, start_sym, grammar, scanner_kw=[]):
self.kw_set = set(scanner_kw)
self.kw_set.add('$')
- #self.R, counter = ruleset(rules(start_sym, grammar, self.kw_set))
self.R = RuleSet(rules(start_sym, grammar, self.kw_set))
self.I = set((r, i) for r in xrange(self.R.rules_count)
for i in xrange(len(self.R[r][1]) + 1))
- #self.rules_count = self.R.rules_count
+ self.precompute_next_items()
self.compute_lr0()
self.LR0 = list(sorted(self.LR0))
self.LR0_idx = {}
@@ -227,19 +226,29 @@ def init_row(self, init=None):
ret[kw] = [] + init
return ret
+ def precompute_next_items(self):
+ self.next_list = dict((k, set()) for k in self.R if type(k) is str)
+ for item in self.I:
+ r, i = item
+ n, e, c = self.R[r]
+ if i > 0 and e[i - 1] in self.next_list:
+ self.next_list[e[i - 1]].add(item)
+
def next_items(self, item, visited=None):
"Compute the yet unvisited items following the given item."
items = set()
if visited is None:
visited = set()
name = self.R[item[0]][0]
- for r, e, i, n in expand_itemset2(self.I, self.R):
- if i > 0 and e[i - 1] == name and (r, i) not in visited:
- visited.add((r, i))
+ for it in self.next_list[name]:
+ if it not in visited:
+ r, i = it
+ e = self.R[r][1]
+ visited.add(it)
if len(e) == i:
- items.update(self.next_items((r, i), visited))
+ items.update(self.next_items(it, visited))
else:
- items.add((r, i))
+ items.add(it)
return items
def following_tokens(self, item):
Please sign in to comment.
Something went wrong with that request. Please try again.