Skip to content

Commit

Permalink
finished testing nonterminal-if-defined and terminal-if-undefined fea…
Browse files Browse the repository at this point in the history 
…tures

git-svn-id: https://slps.svn.sourceforge.net/svnroot/slps@1094 ab42f6e0-554d-0410-b580-99e487e6eeb2
  • Loading branch information
@grammarware
grammarware committed Jun 15, 2011
1 parent 1234890 commit e792b01
Show file tree
Hide file tree
Showing 81 changed files with 2,395 additions and 3 deletions.
39 changes: 36 additions & 3 deletions topics/recovery/hunter/hunter.py
View file
Expand Up @@ -841,6 +841,9 @@ def convert2terminal(x,defd):
if x in defd:
# defined nonterminal
return x
if x[0] == config['start-terminal-symbol'] and x[-1] == config['end-terminal-symbol']:
# already a terminal
return x
if x in metasymbols:
# pseudo-meta-symbol
return x
Expand All @@ -862,12 +865,36 @@ def convert2terminal(x,defd):
if 'nonterminal-if-camelcase' in config.keys() and len(x)>1 and isCamelCase(x):
# configuration claims that CamelCase is a nonterminal
return x
if x[0] == config['start-terminal-symbol'] and x[-1] == config['end-terminal-symbol']:
# already a terminal
return x
# none of the above
return config['start-terminal-symbol'] + x + config['end-terminal-symbol']

def convert2nonterminal(x,defd):
# unfolded for better readability
if len(x) < 4:
# an tiny terminal or even less than that
return x
if x[0] != config['start-terminal-symbol'] or x[-1] != config['end-terminal-symbol']:
# not a terminal at all
return x
y = x[1:-1]
if y in always_terminals:
# configured exception
return x
if 'terminal-if-uppercase' in config.keys() and isUpperCase(y):
# configuration claims that UPPERCASE is a terminal
return x
if 'terminal-if-lowercase' in config.keys() and isLowerCase(y):
# configuration claims that UPPERCASE is a terminal
return x
if 'terminal-if-camelcase' in config.keys() and isCamelCase(y):
# configuration claims that CamelCase is a terminal
return x
if y in defd:
# The moment of truth: could it be a defined nonterminal?
return y
# none of the above
return x

def balanceProd(p):
global debug
i = 2
Expand Down Expand Up @@ -1436,10 +1463,16 @@ def t2nt(tokens,check):
for x in ignore_tokens:
prods = [list(filter(lambda y:y!=x,p)) for p in prods]
#prods = list(map(lambda x:filter(lambda y:y!='\n',x),prods))
# quite usual trick, harmless for most grammars
if 'terminal-if-undefined' in config.keys():
print('STEP 8 (rule 5): turning undefined nonterminals into terminals.')
step8 = True
prods = [[convert2terminal(x,defined) for x in p] for p in prods]
# should be used very carefully because it is common for grammars to have terminals and nonterminals with the "same" name
if 'nonterminal-if-defined' in config.keys():
print('STEP 8 (rule 6): turning terminals that look like defined nonterminals into nonterminals.')
step8 = True
prods = [[convert2nonterminal(x,defined) for x in p] for p in prods]
if 'glue-nonalphanumeric-terminals' in config.keys():
print('STEP 8 (part of rule 3): glueing non-alphanumeric terminal symbols together.')
step8 = True
Expand Down
78 changes: 78 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-lower.bgf
View file
@@ -0,0 +1,78 @@
<?xml version="1.0" encoding="UTF-8"?>
<bgf:grammar xmlns:bgf="http://planet-sl.org/bgf">
<bgf:production>
<nonterminal>foo</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<terminal>BAR</terminal>
</bgf:expression>
<bgf:expression>
<nonterminal>WEZ</nonterminal>
</bgf:expression>
<bgf:expression>
<terminal>FOO</terminal>
</bgf:expression>
<bgf:expression>
<terminal>BAR</terminal>
</bgf:expression>
<bgf:expression>
<terminal>WEZ</terminal>
</bgf:expression>
<bgf:expression>
<terminal>FOO</terminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
<bgf:production>
<nonterminal>Bar</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<terminal>wez</terminal>
</bgf:expression>
<bgf:expression>
<terminal>foo</terminal>
</bgf:expression>
<bgf:expression>
<terminal>bar</terminal>
</bgf:expression>
<bgf:expression>
<terminal>wez</terminal>
</bgf:expression>
<bgf:expression>
<terminal>foo</terminal>
</bgf:expression>
<bgf:expression>
<terminal>bar</terminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
<bgf:production>
<nonterminal>WEZ</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<nonterminal>Foo</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Bar</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Wez</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Foo</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Bar</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Wez</nonterminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
</bgf:grammar>
10 changes: 10 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-lower.edd
View file
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<edd:config xmlns:edd="http://planet-sl.org/edd">
<defining-symbol>:</defining-symbol>
<terminator-symbol>\n</terminator-symbol>
<start-terminal-symbol>“</start-terminal-symbol>
<end-terminal-symbol>”</end-terminal-symbol>
<terminal-if-undefined/>
<nonterminal-if-camelcase/>
<terminal-if-lowercase/>
</edd:config>
3 changes: 3 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-lower.src
View file
@@ -0,0 +1,3 @@
foo : BAR WEZ FOO “BAR” “WEZ” “FOO”
Bar : wez foo bar “wez” “foo” “bar”
WEZ : Foo Bar Wez “Foo” “Bar” “Wez”
78 changes: 78 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-none.bgf
View file
@@ -0,0 +1,78 @@
<?xml version="1.0" encoding="UTF-8"?>
<bgf:grammar xmlns:bgf="http://planet-sl.org/bgf">
<bgf:production>
<nonterminal>foo</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<terminal>BAR</terminal>
</bgf:expression>
<bgf:expression>
<nonterminal>WEZ</nonterminal>
</bgf:expression>
<bgf:expression>
<terminal>FOO</terminal>
</bgf:expression>
<bgf:expression>
<terminal>BAR</terminal>
</bgf:expression>
<bgf:expression>
<terminal>WEZ</terminal>
</bgf:expression>
<bgf:expression>
<terminal>FOO</terminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
<bgf:production>
<nonterminal>Bar</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<terminal>wez</terminal>
</bgf:expression>
<bgf:expression>
<nonterminal>foo</nonterminal>
</bgf:expression>
<bgf:expression>
<terminal>bar</terminal>
</bgf:expression>
<bgf:expression>
<terminal>wez</terminal>
</bgf:expression>
<bgf:expression>
<terminal>foo</terminal>
</bgf:expression>
<bgf:expression>
<terminal>bar</terminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
<bgf:production>
<nonterminal>WEZ</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<nonterminal>Foo</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Bar</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Wez</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Foo</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Bar</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Wez</nonterminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
</bgf:grammar>
9 changes: 9 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-none.edd
View file
@@ -0,0 +1,9 @@
<?xml version="1.0" encoding="UTF-8"?>
<edd:config xmlns:edd="http://planet-sl.org/edd">
<defining-symbol>:</defining-symbol>
<terminator-symbol>\n</terminator-symbol>
<start-terminal-symbol>“</start-terminal-symbol>
<end-terminal-symbol>”</end-terminal-symbol>
<terminal-if-undefined/>
<nonterminal-if-camelcase/>
</edd:config>
3 changes: 3 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-none.src
View file
@@ -0,0 +1,3 @@
foo : BAR WEZ FOO “BAR” “WEZ” “FOO”
Bar : wez foo bar “wez” “foo” “bar”
WEZ : Foo Bar Wez “Foo” “Bar” “Wez”
78 changes: 78 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-upper.bgf
View file
@@ -0,0 +1,78 @@
<?xml version="1.0" encoding="UTF-8"?>
<bgf:grammar xmlns:bgf="http://planet-sl.org/bgf">
<bgf:production>
<nonterminal>foo</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<terminal>BAR</terminal>
</bgf:expression>
<bgf:expression>
<terminal>WEZ</terminal>
</bgf:expression>
<bgf:expression>
<terminal>FOO</terminal>
</bgf:expression>
<bgf:expression>
<terminal>BAR</terminal>
</bgf:expression>
<bgf:expression>
<terminal>WEZ</terminal>
</bgf:expression>
<bgf:expression>
<terminal>FOO</terminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
<bgf:production>
<nonterminal>Bar</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<terminal>wez</terminal>
</bgf:expression>
<bgf:expression>
<nonterminal>foo</nonterminal>
</bgf:expression>
<bgf:expression>
<terminal>bar</terminal>
</bgf:expression>
<bgf:expression>
<terminal>wez</terminal>
</bgf:expression>
<bgf:expression>
<terminal>foo</terminal>
</bgf:expression>
<bgf:expression>
<terminal>bar</terminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
<bgf:production>
<nonterminal>WEZ</nonterminal>
<bgf:expression>
<sequence>
<bgf:expression>
<nonterminal>Foo</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Bar</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Wez</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Foo</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Bar</nonterminal>
</bgf:expression>
<bgf:expression>
<nonterminal>Wez</nonterminal>
</bgf:expression>
</sequence>
</bgf:expression>
</bgf:production>
</bgf:grammar>
10 changes: 10 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-upper.edd
View file
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?>
<edd:config xmlns:edd="http://planet-sl.org/edd">
<defining-symbol>:</defining-symbol>
<terminator-symbol>\n</terminator-symbol>
<start-terminal-symbol>“</start-terminal-symbol>
<end-terminal-symbol>”</end-terminal-symbol>
<terminal-if-undefined/>
<nonterminal-if-camelcase/>
<terminal-if-uppercase/>
</edd:config>
3 changes: 3 additions & 0 deletions topics/recovery/hunter/tests/8-nt2t-camel-upper.src
View file
@@ -0,0 +1,3 @@
foo : BAR WEZ FOO “BAR” “WEZ” “FOO”
Bar : wez foo bar “wez” “foo” “bar”
WEZ : Foo Bar Wez “Foo” “Bar” “Wez”

0 comments on commit e792b01

Please sign in to comment.