From 7880de169102a72fbac2bed1be816278c537c69b Mon Sep 17 00:00:00 2001 From: grammarware Date: Wed, 3 Sep 2008 19:24:59 +0000 Subject: [PATCH] LCI v1.11: tree extraction, diffing, coupled transformations git-svn-id: https://slps.svn.sourceforge.net/svnroot/slps@234 ab42f6e0-554d-0410-b580-99e487e6eeb2 --- topics/convergence/lci/lci.py | 143 ++++++++++++++++++++------- topics/extraction/ldf2set/ldf2set.py | 53 +++++----- topics/fl/lci/fl.lcf | 5 + 3 files changed, 140 insertions(+), 61 deletions(-) diff --git a/topics/convergence/lci/lci.py b/topics/convergence/lci/lci.py index 469d7aae..f71ba212 100755 --- a/topics/convergence/lci/lci.py +++ b/topics/convergence/lci/lci.py @@ -10,6 +10,7 @@ testsets = {} tester = {} extractor = {} +treeextractor = {} targets = {} parser = {} evaluator = {} @@ -49,41 +50,44 @@ def sysexit(n): def readxmlconfig (cfg): config = ElementTree.parse(cfg) # shortcuts - for outline in config.findall('//shortcut'): - shortcuts[outline.findtext('name')]=expandxml(outline.findall('expansion')[0],{}) + for xmlnode in config.findall('//shortcut'): + shortcuts[xmlnode.findtext('name')]=expandxml(xmlnode.findall('expansion')[0],{}) # actions - for outline in config.findall('//target/branch/perform'): - if outline.text not in actions: - actions.append(outline.text) + for xmlnode in config.findall('//target/branch/perform'): + if xmlnode.text not in actions: + actions.append(xmlnode.text) # testset - for outline in config.findall('//testset'): - testsets[outline.findtext('name')]=expandxml(outline.findall('command')[0],{}) + for xmlnode in config.findall('//testset'): + testsets[xmlnode.findtext('name')]=expandxml(xmlnode.findall('command')[0],{}) # sources - for outline in config.findall('//source'): - extractor[outline.findtext('name')]=expandxml(outline.findall('grammar/extraction')[0],{}) - pcmd = ecmd = '' - if outline.findall('grammar/parsing'): - parser[outline.findtext('name')]=expandxml(outline.findall('grammar/parsing')[0],{}) - if outline.findall('grammar/evaluation'): - evaluator[outline.findtext('name')]=expandxml(outline.findall('grammar/evaluation')[0],{}) + for xmlnode in config.findall('//source'): + extractor[xmlnode.findtext('name')]=expandxml(xmlnode.findall('grammar/extraction')[0],{}) + if xmlnode.findall('grammar/parsing'): + parser[xmlnode.findtext('name')]=expandxml(xmlnode.findall('grammar/parsing')[0],{}) + if xmlnode.findall('grammar/evaluation'): + evaluator[xmlnode.findtext('name')]=expandxml(xmlnode.findall('grammar/evaluation')[0],{}) + if xmlnode.findall('tree/extraction'): + treeextractor[xmlnode.findtext('name')]=expandxml(xmlnode.findall('tree/extraction')[0],{}) + if xmlnode.findall('tree/evaluation'): + treeevaluator[xmlnode.findtext('name')]=expandxml(xmlnode.findall('tree/evaluation')[0],{}) tmp = [] - for set in outline.findall('testing/set'): + for set in xmlnode.findall('testing/set'): tmp.append(set.text) - tester[outline.findtext('name')]=tmp[:] + tester[xmlnode.findtext('name')]=tmp[:] # targets - for outline in config.findall('//target'): - name = outline.findtext('name') + for xmlnode in config.findall('//target'): + name = xmlnode.findtext('name') targets[name]= [[],''] - for br in outline.findall('branch'): + for br in xmlnode.findall('branch'): branch = [br.findtext('input')] for p in br.findall('perform'): branch.append(p.text) targets[name][0].append(branch) # tools - for outline in config.findall('//tool'): - tools[outline.findtext('name')] = expandxml(outline.findall('grammar')[0],{}) - if outline.findall('tree'): - treetools[outline.findtext('name')] = expandxml(outline.findall('tree')[0],{}) + for xmlnode in config.findall('//tool'): + tools[xmlnode.findtext('name')] = expandxml(xmlnode.findall('grammar')[0],{}) + if xmlnode.findall('tree'): + treetools[xmlnode.findtext('name')] = expandxml(xmlnode.findall('tree')[0],{}) print 'Read',len(shortcuts),'shortcuts,',`len(tools)`+'+'+`len(treetools)`,'tools,',len(actions),'actions,',len(targets),'targets,' print len(testsets),'test sets,',len(extractor),'sources:',len(parser),'parsers &',len(evaluator),'evaluators' @@ -238,6 +242,7 @@ def extractall(): if os.access('snapshot/'+bgf+'.bgf',os.R_OK): print 'Rolled back to the saved version, proceeding...' copyfile('snapshot/'+bgf+'.bgf','bgf/'+bgf+'.bgf') + logwrite('cp snapshot/'+bgf+'.bgf bgf/'+bgf+'.bgf') almostfailed.append(bgf) else: failednode.append(bgf) @@ -294,7 +299,7 @@ def preparebgf(cut): elif tools.has_key('validation'): a = tools['validation']+' bgf/'+curname+'.bgf' logwrite(a) - print 'Performed',name,'- the result is', + print 'Performed',name,'-', if os.system(a+shutup): print 'NOT', print 'valid' @@ -302,8 +307,8 @@ def preparebgf(cut): print 'Performed',name return curname -def buildtargets(): - unordered = targets.keys() +def ordertargets(): + unordered = targets.keys()[:] ordered = [] while len(unordered): for t in unordered: @@ -314,7 +319,10 @@ def buildtargets(): if flag: ordered.append(t) unordered.remove(t) - for t in ordered: + return ordered + +def buildtargets(): + for t in ordertargets(): inputs = targets[t][0] fileinputs = ['']*len(inputs) for i in range(0,len(inputs)): @@ -356,6 +364,52 @@ def diffall(t,car,cdr): diffall(t,car,[head]) diffall(t,cdr[0],cdr[1:]) +def chainXBTF(testcase,steps,t): + fr = testcase + for step in steps: + if step==steps[-1]: + # name it after the target + re = fr.split('.')[0]+'.'+t+'.btf' + else: + # name it as input.transformationName.btf + re = '.'.join(fr.split('.')[:-1])+'.'+step+'.btf' + run = treetools['transformation']+' xbgf/'+step+'.xbgf '+fr+' '+re + logwrite(run) + #print 'Performing coupled',step,'on',fr,'-', + if os.system(run+shutup): + print 'Performing coupled',step,'on',ft,'failed' + break + fr = re + tmp = steps[:] + tmp.reverse() + print 'Performed coupled',' '.join(tmp),'on',testcase, + if treetools.has_key('validation'): + run = treetools['validation']+' '+re + if os.system(run+shutup): + print '- NOT valid', + else: + print '- valid' + else: + print + +def diffBTFs(t): + if len(testsets)<2: + # with one test set there's nothing to diff + return + if not treetools.has_key('comparison'): + # no tree diff tool specified + rturn + basetestset = testsets.keys()[0] + for basetestcase in glob.glob(basetestset+'/*.'+t+'.btf'): + for testset in testsets.keys()[1:]: + for testcase in glob.glob(testset+'/'+basetestcase.split('/')[1]): + run = treetools['comparison']+' '+basetestcase+' '+testcase + print 'Found',basetestcase.split('/')[1],'in',basetestset,'and',testset,'- they', + if os.system(run+shutup): + print 'DIFFER' + else: + print 'match' + def runtestset(): for testset in testsets.keys(): # extracting @@ -366,6 +420,30 @@ def runtestset(): print 'could not be extracted' continue print 'extracted' + for src in treeextractor.keys(): + for testset in tester[src]: + for testcase in glob.glob(testset+'/*.src'): + run = treeextractor[src]+' '+testcase+' '+testcase+'.btf' + logwrite(run) + print 'Tree extraction from',testcase, + if os.system(run+shutup): + print 'failed' + else: + print 'completed' + for t in ordertargets(): + for branch in targets[t][0]: + if treeextractor.has_key(branch[0]): + # it's a source, let's check it we have an extracted tree + for testset in tester[branch[0]]: + for testcase in glob.glob(testset+'/*.src.btf'): + chainXBTF(testcase,branch[1:],t) + if targets.has_key(branch[0]): + # it's a target, let's see if we have any test cases arrived at it + for testset in testsets.keys(): + for testcase in glob.glob(testset+'/*.'+branch[0]+'.btf'): + chainXBTF(testcase,branch[1:],t) + diffBTFs(t) + for testset in testsets.keys(): # testing parser for testcase in glob.glob(testset+'/*.src'): results={} @@ -416,7 +494,7 @@ def checkconsistency(): #sysexit(8) if __name__ == "__main__": - print 'Language Covergence Infrastructure v1.10alpha' + print 'Language Covergence Infrastructure v1.11' if len(sys.argv) == 3: log = open(sys.argv[1].split('.')[0]+'.log','w') readxmlconfig(sys.argv[1]) @@ -426,21 +504,18 @@ def checkconsistency(): if tools.has_key('validation'): validateall() buildtargets() - print 'Grammar convergence finished.' + print '>>> Grammar convergence phase finished.' if testsets: runtestset() - print 'Testing finished.' + print '>>> Tree convergence phase finished.' else: print 'No testing performed.' dumpgraph(sys.argv[2]) - #print failednode - #print failedarc - #print almostfailed if failednode or failedarc: sysexit(100) log.close() else: print 'Usage:' - print ' ',sys.argv[0],'','' + print ' ',sys.argv[0],'','' sysexit(1) diff --git a/topics/extraction/ldf2set/ldf2set.py b/topics/extraction/ldf2set/ldf2set.py index 879b7a2c..3d9bfc05 100755 --- a/topics/extraction/ldf2set/ldf2set.py +++ b/topics/extraction/ldf2set/ldf2set.py @@ -13,50 +13,49 @@ def unpacksamples(where,dir): library={} cx = 0 tree = ElementTree.parse(where) - for outline in tree.findall("//sample"): + for xmlnode in tree.findall("//sample"): cx+=1 - if outline.attrib.has_key('id'): - name = outline.attrib['id'] + if xmlnode.attrib.has_key('id'): + name = xmlnode.attrib['id'] + library[name]=xmlnode.text else: name = 'untitled'+`cx` - torun = open (dir+'/'+name+'.src',"w") - for line in outline.text.split('\n'): - if line.strip()!='': - torun.write(line.strip()+'\n') - torun.close() - if outline.attrib.has_key('id'): - library[name]=outline.text - if outline.attrib.has_key('sort'): - sort=outline.attrib['sort'] + if xmlnode.attrib.has_key('sort'): + print 'Test case',name,'of sort',xmlnode.attrib['sort'],'not extracted.' else: - sort=None + src = open (dir+'/'+name+'.src',"w") + for line in xmlnode.text.split('\n'): + if line.strip()!='': + src.write(line.strip()+'\n') + src.close() + print 'Test case',name,'extracted.' # All executions - for outline in tree.findall("//runnable"): + for xmlnode in tree.findall("//runnable"): cx+=1 - if outline.attrib.has_key('id'): - name = outline.attrib['id'] + if xmlnode.attrib.has_key('id'): + name = xmlnode.attrib['id'] else: name = 'untitled'+`cx` - if outline.findtext('context'): - if not library.has_key(outline.findtext('context')): - print "No context found for sample",name,'('+outline.findtext('context')+'), test case not used' + if xmlnode.findtext('context'): + if not library.has_key(xmlnode.findtext('context')): + print "No context found for sample",name,'('+xmlnode.findtext('context')+'), test case not used' continue else: con = open (dir+'/'+name+'.src','w') - for line in library[outline.findtext('context')].split('\n'): + for line in library[xmlnode.findtext('context')].split('\n'): if line.strip()!='': con.write(line.strip()+'\n') con.close() - if outline.findtext('yields'): + if xmlnode.findtext('yields'): con = open (dir+'/'+name+'.val','w') - con.write(outline.findtext('yields')) + con.write(xmlnode.findtext('yields')) con.close() - torun = open (dir+'/'+name+'.run','w') - line = outline.findtext('main') - for arg in outline.findall("argument"): + run = open (dir+'/'+name+'.run','w') + line = xmlnode.findtext('main') + for arg in xmlnode.findall("argument"): line += ' ' + arg.text - torun.write(line+'\n') - torun.close() + run.write(line+'\n') + run.close() print cx,'samples in the test set.' if __name__ == "__main__": diff --git a/topics/fl/lci/fl.lcf b/topics/fl/lci/fl.lcf index 7e9760d0..9c0bbe9f 100644 --- a/topics/fl/lci/fl.lcf +++ b/topics/fl/lci/fl.lcf @@ -118,6 +118,11 @@ wrappers/runasfsdfevaluator + + + tools/asfix2btf bgf/sdf.bgf + + codesamples