From 3af427a54872a4f614e755e0159cfa60f8aaf135 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 09:52:43 +0100 Subject: [PATCH 01/72] Fix analyses/derivative.py --- .gitignore | 2 +- analyses/derivative.py | 558 +++++++++++++++++++++-------------------- lib/cpplib.py | 2 +- setup.py | 8 +- 4 files changed, 294 insertions(+), 276 deletions(-) diff --git a/.gitignore b/.gitignore index 5d76ddc..1609f28 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ tmp/ projects/ ## virtual environment virtualenv/ - +venv/ ### cppstats temporary files ## sample cppstats list-input diff --git a/analyses/derivative.py b/analyses/derivative.py index 77122f5..b1781b1 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -29,8 +29,8 @@ import os import re import sys -import xmlrpclib from argparse import ArgumentParser, RawTextHelpFormatter +from functools import reduce # ################################################# # path adjustments, so that all imports can be done relative to these paths @@ -38,21 +38,20 @@ __lib_subfolder = "lib" sys.path.append(os.path.abspath(__lib_subfolder)) # lib subfolder - # ################################################# # external modules # enums from enum import Enum - # python-lxml module +# python-lxml module from lxml import etree - # statistics module -from statlib import pstat +# statistics module +import numpy as np # pyparsing module import pyparsing as pypa -pypa.ParserElement.enablePackrat() # speed up parsing -sys.setrecursionlimit(8000) # handle larger expressions +pypa.ParserElement.enablePackrat() # speed up parsing +sys.setrecursionlimit(8000) # handle larger expressions ################################################## # config: @@ -77,28 +76,30 @@ __conditionals_else = ['else'] __conditionals_endif = ['endif'] __conditionals_all = __conditionals + __conditionals_elif + \ - __conditionals_else + __conditionals_else __macro_define = ['define'] -__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" -__curfile = '' # current processed xml-file -__defset = set() # macro-objects -__defsetf = dict() # macro-objects per file +__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", +# used as "GLIBVERSION(x,y,z) 100*x+10*y+z" +__curfile = '' # current processed xml-file +__defset = set() # macro-objects +__defsetf = dict() # macro-objects per file + # collected statistics class __statsorder(Enum): - FILENAME = 0 # name of the file - NUMANNOTATIONS = 1 # number of all annotations - NOIANNOTATIONS = 2 # number of interacting annotations - # example: A&B&C&D --> A&B, B&C, B&D - # example: A&B&C&D -/> A&B, B&C, A&C + FILENAME = 0 # name of the file + NUMANNOTATIONS = 1 # number of all annotations + NOIANNOTATIONS = 2 # number of interacting annotations + # example: A&B&C&D --> A&B, B&C, B&D + # example: A&B&C&D -/> A&B, B&C, A&C + ################################################## ################################################## # helper functions, constants and errors -def returnFileNames(folder, extfilt = ['.xml']): +def returnFileNames(folder, extfilt=['.xml']): '''This function returns all files of the input folder and its subfolders.''' filesfound = list() @@ -111,24 +112,26 @@ def returnFileNames(folder, extfilt = ['.xml']): wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) + tmpfiles) tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles) filesfound += tmpfiles tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders) wqueue += tmpfolders return filesfound + def uniqueItems(l): l = sorted(l) return list(k for k, _ in itertools.groupby(l)) + def _flatten(l): """This function takes a list as input and returns a flatten version of the list. So all nested lists are unpacked and moved up to the @@ -141,7 +144,7 @@ def _flatten(l): i -= 1 break else: - l[i:i+1] = l[i] + l[i:i + 1] = l[i] i += 1 return l @@ -153,10 +156,10 @@ def _collectDefines(d): but not #define GLIBCVER(x,y,z) ... """ __defset.add(d[0]) - if __defsetf.has_key(__curfile): + if __curfile in __defsetf: __defsetf[__curfile].add(d[0]) else: - __defsetf[__curfile] = set([d[0]]) + __defsetf[__curfile] = {d[0]} return d @@ -172,41 +175,42 @@ def _collectDefines(d): __string = pypa.QuotedString('\'', '\\') __hexadec = \ - pypa.Literal('0x').suppress() + \ - pypa.Word(pypa.hexnums).\ - setParseAction(lambda t: str(int(t[0], 16))) + \ - pypa.Optional(__numlitu) + \ - pypa.Optional(__numlitl) + \ - pypa.Optional(__numlitl) + pypa.Literal('0x').suppress() + \ + pypa.Word(pypa.hexnums). setParseAction(lambda t: str(int(t[0], 16))) + \ + pypa.Optional(__numlitu) + \ + pypa.Optional(__numlitl) + \ + pypa.Optional(__numlitl) __integer = \ - pypa.Optional('~') + \ - pypa.Word(pypa.nums+'-') + \ - pypa.Optional(__numlitu) + \ - pypa.Optional(__numlitl) + \ - pypa.Optional(__numlitl) + pypa.Optional('~') + \ + pypa.Word(pypa.nums + '-') + \ + pypa.Optional(__numlitu) + \ + pypa.Optional(__numlitl) + \ + pypa.Optional(__numlitl) __identifier = \ - pypa.Word(pypa.alphanums+'_'+'-'+'@'+'$') # @ not allowed but they do occur -__arg = pypa.Word(pypa.alphanums+'_') -__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + \ - __arg) + pypa.Word(pypa.alphanums + '_' + '-' + '@' + '$') # @ not allowed but they do occur +__arg = pypa.Word(pypa.alphanums + '_') +__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + __arg) __fname = pypa.Word(pypa.alphas + '_' + '#', pypa.alphanums + '_' + '#') -__function = pypa.Group(__fname + pypa.Literal('(').suppress() + \ - __args + pypa.Literal(')').suppress()) +__function = pypa.Group(__fname + pypa.Literal('(').suppress() + __args + pypa.Literal(')').suppress()) class NoEquivalentSigError(Exception): def __init__(self): pass + def __str__(self): - return ("No equivalent signature found!") + return "No equivalent signature found!" + class IfdefEndifMismatchError(Exception): def __init__(self): pass + def __str__(self): - return ("Ifdef and endif do not match!") + return "Ifdef and endif do not match!" + ################################################## @@ -221,22 +225,25 @@ def _collapseSubElementsToList(node): return ''.join([it for it in itdesc]) - def _parseFeatureSignature(sig): """This function parses a given feature-signature.""" mal = set() - def _rewriteOne(p): return '' - def _rewriteTwo(p): return '' - def _addIdentifier2Mal(p): mal.add(p[0]) + def _rewriteOne(p): + return '' + + def _rewriteTwo(p): + return '' - operand = __string | __hexadec | __function | __integer | \ - __identifier.setParseAction(_addIdentifier2Mal) + def _addIdentifier2Mal(p): + mal.add(p[0]) + + operand = __string | __hexadec | __function | __integer | __identifier.setParseAction(_addIdentifier2Mal) compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / % & | << >>') expr = pypa.operatorPrecedence(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), - ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), + ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), (compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), ('&&', 2, pypa.opAssoc.LEFT, _rewriteTwo), @@ -245,14 +252,14 @@ def _addIdentifier2Mal(p): mal.add(p[0]) try: rsig = expr.parseString(sig)[0] - except pypa.ParseException, e: + except pypa.ParseException as e: print('ERROR (parse): cannot parse sig (%s) -- (%s)' % - (sig, e.col)) + (sig, e.col)) return sig except RuntimeError: - print('ERROR (time): cannot parse sig (%s)' % (sig)) + print(f'ERROR (time): cannot parse sig {sig}') return sig - return (mal, ''.join(rsig)) + return mal, ''.join(rsig) def _getMacroSignature(ifdefnode): @@ -271,9 +278,9 @@ def _getMacroSignature(ifdefnode): # get either the expr or the name tag, # which is always the second descendant - if (tag in ['if', 'elif', 'ifdef', 'ifndef']): + if tag in ['if', 'elif', 'ifdef', 'ifndef']: nexpr = [itex for itex in ifdefnode.iterdescendants()] - if (len(nexpr) == 1): + if len(nexpr) == 1: res = nexpr[0].tail else: nexpr = nexpr[1] @@ -287,7 +294,7 @@ def _prologCSV(folder): fd = open(os.path.join(folder, __outputfile), 'w') fdcsv = csv.writer(fd, delimiter=',') fdcsv.writerow(__statsorder.__members__.keys()) - return (fd, fdcsv) + return fd, fdcsv def _countNestedIfdefs(root): @@ -299,23 +306,24 @@ def _countNestedIfdefs(root): for elem in elements: ns, tag = __cpprens.match(elem.tag).groups() - if ((tag in __conditionals_endif) - and (ns == __cppnscpp)): cncur -= 1 - if ((tag in __conditionals) - and (ns == __cppnscpp)): + if tag in __conditionals_endif and ns == __cppnscpp: + cncur -= 1 + if tag in __conditionals and ns == __cppnscpp: cncur += 1 cnlist.append(cncur) - if (len(cnlist) > 0): + if len(cnlist) > 0: nnimax = max(cnlist) nnitmp = filter(lambda n: n > 0, cnlist) - nnimean = pstat.stats.lmean(nnitmp) + nnimean = np.mean(nnitmp) else: nnimax = 0 nnimean = 0 - if (len(cnlist) > 1): nnistd = pstat.stats.lstdev(cnlist) - else: nnistd = 0 - return (nnimax, nnimean, nnistd) + if len(cnlist) > 1: + nnistd = np.std(cnlist) + else: + nnistd = 0 + return nnimax, nnimean, nnistd def _getFeatureSignature(condinhist): @@ -325,7 +333,7 @@ def _getFeatureSignature(condinhist): # signature; reason is elements like else or elif, which mean # basically invert the fname found before # rewritelist = [(tag, fname, )] - rewritelist = [None]*len(condinhist) + rewritelist = [None] * len(condinhist) cur = -1 for tag, fname in condinhist: @@ -333,8 +341,8 @@ def _getFeatureSignature(condinhist): if tag == 'if': rewritelist[cur] = (tag, fname, False) if tag in ['elif', 'else']: - (t, f, _) = rewritelist[cur-1] - rewritelist[cur-1] = (t, f, True) + (t, f, _) = rewritelist[cur - 1] + rewritelist[cur - 1] = (t, f, True) rewritelist[cur] = (tag, fname, False) fsig = '' @@ -372,7 +380,7 @@ def _getASTFuture(node): node itself.""" dess = [] - while (node is not None): + while node is not None: dess += [sib for sib in node.itersiblings(preceding=False)] node = node.getparent() @@ -422,58 +430,57 @@ def _getFeatures(root): """ def _wrapGrOuterUp(fouter, featuresgrouter, eelem): - itouter = fouter[-1] # feature surround tags + itouter = fouter[-1] # feature surround tags fouter = fouter[:-1] selem = itouter[0][1] - for (sig, _) in itouter: + for sig, _ in itouter: featuresgrouter.append((sig, selem, eelem)) - return (fouter, featuresgrouter) - + return fouter, featuresgrouter def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # wrap up the feature - if (not flist): + if not flist: raise IfdefEndifMismatchError() - itsig = flist[-1] # feature signature + itsig = flist[-1] # feature signature flist = flist[:-1] - itcode = fcode[-1] # feature code + itcode = fcode[-1] # feature code itcode = itcode.replace('\n\n', '\n') - itcode = itcode[1:] # itcode starts with '\n'; del + itcode = itcode[1:] # itcode starts with '\n'; del fcode = fcode[:-1] - itinner = finner[-1] # feature enclosed tags + itinner = finner[-1] # feature enclosed tags finner = finner[:-1] # handle the feature code - if (features.has_key(itsig)): + if features.has_key(itsig): features[itsig][1].append(itcode) else: - features[itsig] = (len(flist)+1, [itcode]) + features[itsig] = (len(flist) + 1, [itcode]) # handle the inner granularity featuresgrinner.append((itsig, itinner)) - return (features, featuresgrinner, fcode, flist, finner) - - features = {} # see above; return value - featuresgrinner = [] # see above; return value - featuresgrouter = [] # see above; return value - flist = [] # holds the features in order - # list empty -> no features to parse - # list used as a stack - # last element = top of stack; - # and the element we currently - # collecting source-code lines for - fouter = [] # holds the xml-nodes of the ifdefs/endifs - # in order like flist - fcode = [] # holds the code of the features in - # order like flist - finner = [] # holds the tags of the features in - # order like flist - condinhist = [] # order of the conditional includes - # with feature names - parcon = False # parse-conditional-flag - parend = False # parse-endif-flag - _ = 0 # else and elif depth + return features, featuresgrinner, fcode, flist, finner + + features = {} # see above; return value + featuresgrinner = [] # see above; return value + featuresgrouter = [] # see above; return value + flist = [] # holds the features in order + # list empty -> no features to parse + # list used as a stack + # last element = top of stack; + # and the element we currently + # collecting source-code lines for + fouter = [] # holds the xml-nodes of the ifdefs/endifs + # in order like flist + fcode = [] # holds the code of the features in + # order like flist + finner = [] # holds the tags of the features in + # order like flist + condinhist = [] # order of the conditional includes + # with feature names + parcon = False # parse-conditional-flag + parend = False # parse-endif-flag + _ = 0 # else and elif depth # iterate over all tags separately - and -tag for event, elem in etree.iterwalk(root, events=("start", "end")): @@ -483,13 +490,13 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # hitting on conditional-macro if ((tag in __conditionals_all) and (event == 'start') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = True # hitting next conditional macro; any of ifdef, else or elif if ((tag in __conditionals_all) and (event == 'end') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = False # with else or elif we finish up the last if, therefor @@ -497,30 +504,29 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): if ((tag in __conditionals_else) or (tag in __conditionals_elif)): (features, featuresgrinner, - fcode, flist, finner) = _wrapFeatureUp(features, - featuresgrinner, fcode, flist, finner) + fcode, flist, finner) = _wrapFeatureUp(features, + featuresgrinner, fcode, flist, finner) fname = _getMacroSignature(elem) - if fname: condinhist.append((tag, fname)) - else: condinhist.append((tag, '')) + if fname: + condinhist.append((tag, fname)) + else: + condinhist.append((tag, '')) fsig = _getFeatureSignature(condinhist) - if (tag in __conditionals): fouter.append([]) + if tag in __conditionals: + fouter.append([]) fouter[-1] += ([(fsig, elem)]) flist.append(fsig) fcode.append('') finner.append([]) # hitting end-tag of elif-macro - if ((tag in __conditionals_elif) - and (event == 'end') - and (ns == __cppnscpp)): + if tag in __conditionals_elif and event == 'end' and ns == __cppnscpp: parcon = False # hitting end-tag of define-macro - if ((tag in __macro_define) \ - and (event == 'end') \ - and (ns == __cppnscpp)): + if tag in __macro_define and event == 'end' and ns == __cppnscpp: _parseAndAddDefine(elem) # iterateting in subtree of conditional-node @@ -529,24 +535,24 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # handling endif-macro # hitting an endif-macro start-tag - if ((tag in __conditionals_endif) \ - and (event == "start") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if ((tag in __conditionals_endif) + and (event == "start") + and (ns == __cppnscpp)): # check the cpp:namespace parend = True # hitting the endif-macro end-tag - if ((tag in __conditionals_endif) \ - and (event == "end") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if ((tag in __conditionals_endif) + and (event == "end") + and (ns == __cppnscpp)): # check the cpp:namespace parend = False (features, featuresgrinner, fcode, flist, finner) = \ _wrapFeatureUp(features, featuresgrinner, - fcode, flist, finner) + fcode, flist, finner) (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, - featuresgrouter, elem) + featuresgrouter, elem) - while (condinhist[-1][0] != 'if'): + while condinhist[-1][0] != 'if': condinhist = condinhist[:-1] condinhist = condinhist[:-1] @@ -555,18 +561,18 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): continue # collect the source-code of the feature - if (len(flist)): - if ((event == "start") and (elem.text)): + if len(flist): + if event == "start" and elem.text: fcode[-1] += elem.text - if ((event == "end") and (elem.tail)): + if event == "end" and elem.tail: fcode[-1] += elem.tail - if (ns == __cppnsdef or tag not in __conditionals_all): + if ns == __cppnsdef or tag not in __conditionals_all: finner[-1].append((tag, event, elem.sourceline)) - if (flist): + if flist: raise IfdefEndifMismatchError() - return (features, featuresgrinner, featuresgrouter) + return features, featuresgrinner, featuresgrouter def _getOuterGranularity(fnodes): @@ -577,7 +583,7 @@ def _getOuterGranularity(fnodes): grouter = list() for (sig, selem, _) in fnodes: - tags = _getASTHistory(selem)[:-1] # cut of unit-tag + tags = _getASTHistory(selem)[:-1] # cut of unit-tag grouter.append((sig, tags, selem.sourceline)) return grouter @@ -593,93 +599,102 @@ def _getOuterGranularityStats(lgran): """ gotopbgr = 0 gofunbgr = 0 - gostrbrl = 0 # local - gostrbrg = 0 # global + gostrbrl = 0 # local + gostrbrg = 0 # global goinnbgr = 0 goexpbgr = 0 gostmbgr = 0 gopambgr = 0 - goerror = 0 + goerror = 0 for (_, gran, line) in lgran: if len(gran) == 0: gotopbgr += 1 continue if gran[0] in ['block']: - if len(gran) == 1: # configure the method signature + if len(gran) == 1: # configure the method signature gofunbgr += 1 continue if gran[1] in ['function', 'extern', 'block']: gofunbgr += 1 elif gran[1] in ['struct', 'union', - 'enum']: # test_struct_union_enum.c - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + 'enum']: # test_struct_union_enum.c + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 elif gran[1] in ['expr']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 elif gran[1] in ['while', 'for', 'then', 'do', - 'else', 'switch', 'case', 'default']: - goinnbgr += 1 # test_loop.c - elif gran[1] in ['decl']: # test_struct_union_enum.c - if 'function' in gran[3:]: gostrbrl += 1 - else: gostrbrg += 1 + 'else', 'switch', 'case', 'default']: + goinnbgr += 1 # test_loop.c + elif gran[1] in ['decl']: # test_struct_union_enum.c + if 'function' in gran[3:]: + gostrbrl += 1 + else: + gostrbrg += 1 else: print('ERROR: gran (%s) at this ' - 'level unknown (line %s)' % (gran, line)) + 'level unknown (line %s)' % (gran, line)) goerror += 1 continue elif gran[0] in ['expr']: - if gran[1] in ['expr_stmt']: # test_stmt.c + if gran[1] in ['expr_stmt']: # test_stmt.c gostmbgr += 1 - elif gran[1] in ['condition', 'return']: # test_condition.c + elif gran[1] in ['condition', 'return']: # test_condition.c goexpbgr += 1 - elif gran[1] in ['argument']: # test_call.c + elif gran[1] in ['argument']: # test_call.c gostmbgr += 1 elif gran[1] in ['block']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 - elif gran[1] in ['init', 'index']: # test_stmt.c + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 + elif gran[1] in ['init', 'index']: # test_stmt.c gostmbgr += 1 else: print('ERROR: gran (%s) at this level' - 'unknown (line %s)' % (gran, line)) + 'unknown (line %s)' % (gran, line)) goerror += 1 continue - elif gran[0] in ['while', 'do']: # test_loop.c + elif gran[0] in ['while', 'do']: # test_loop.c goinnbgr += 1 continue elif gran[0] in ['expr_stmt'] and len(gran) == 1: gostmbgr += 1 continue elif gran[:3] == ['expr_stmt', 'block', 'struct']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 continue - elif gran[0] in ['decl_stmt']: # test_stmt.c + elif gran[0] in ['decl_stmt']: # test_stmt.c gostmbgr += 1 continue - elif gran[0] in ['condition']: # test_condition.c + elif gran[0] in ['condition']: # test_condition.c goexpbgr += 1 continue elif gran[0] in ['if', 'else', 'case', 'default', - 'then', 'for']: # test_condition.c + 'then', 'for']: # test_condition.c goinnbgr += 1 continue elif gran[0] in ['parameter_list', - 'argument_list']: # test_call.c + 'argument_list']: # test_call.c gopambgr += 1 continue elif gran[0] in ['argument'] and gran[1] in ['argument_list']: gostmbgr += 1 - elif gran[0] in ['init'] and gran[1] in ['decl']: # test_stmt.c + elif gran[0] in ['init'] and gran[1] in ['decl']: # test_stmt.c gostmbgr += 1 continue - elif gran[0] in ['function']: # function prototype + elif gran[0] in ['function']: # function prototype continue else: - print('ERROR: outer granularity (%s, %s) not recognize!' % \ - (gran, line)) + print(f'ERROR: outer granularity ({gran}, {line}) not recognize!') goerror += 1 return (gotopbgr, gofunbgr, gostrbrl, gostrbrg, @@ -707,24 +722,25 @@ def _getInnerGranularityStats(igran): skiptilltag = '' - for (_, gran) in igran: - for (tag, event, line) in gran: - if (skiptilltag != ''): + for _, gran in igran: + for tag, event, line in gran: + if skiptilltag != '': if (tag == skiptilltag[0] and event == 'end' and line == skiptilltag[2]): skiptilltag = '' continue - if tag in ['name', 'endif']: continue + if tag in ['name', 'endif']: + continue elif tag in ['define', 'directive', 'include', - 'macro', 'undef']: + 'macro', 'undef']: gmacrogr += 1 elif tag in ['struct', 'union', 'enum', 'function', 'extern', - 'function_decl', 'decl_stmt', 'typedef']: + 'function_decl', 'decl_stmt', 'typedef']: ginablgr += 1 elif tag in ['if', 'while', 'return', 'then', - 'for', 'do', 'case', 'else', 'block']: + 'for', 'do', 'case', 'else', 'block']: giunblgr += 1 elif tag in ['param', 'argument']: gipambgr += 1 @@ -736,12 +752,13 @@ def _getInnerGranularityStats(igran): gistmbgr += 1 else: print('ERROR: inner granularity (%s, %s, %s)) ' - 'not recognized!' % (tag, event, line)) + 'not recognized!' % (tag, event, line)) gierror += 1 continue - if event == 'start': skiptilltag = (tag, event, line) + if event == 'start': + skiptilltag = (tag, event, line) - return (gmacrogr, ginablgr, giunblgr, giexpbgr, gistmbgr, gierror) + return gmacrogr, ginablgr, giunblgr, giexpbgr, gistmbgr, gierror def _getFeatureStats(features): @@ -757,30 +774,30 @@ def _getFeatureStats(features): """ lof = 0 nod = 0 - lofmin = -1 # feature-code can be empty + lofmin = -1 # feature-code can be empty lofmax = 0 lofmean = 0 lofstd = 0 nof = len(features.keys()) tmp = [item for (_, item) in features.itervalues()] tmp = _flatten(tmp) - floflist = map(lambda n: n.count('\n'), tmp) + floflist = list(map(lambda n: n.count('\n'), tmp)) - if (len(floflist)): + if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = reduce(lambda m,n: m+n, floflist) - lofmean = pstat.stats.lmean(floflist) + lof = reduce(lambda m, n: m + n, floflist) + lofmean = np.mean(floflist) - if (len(floflist) > 1): - lofstd = pstat.stats.lstdev(floflist) + if len(floflist) > 1: + lofstd = np.std(floflist) - return (nof, nod, lof, lofmin, lofmax, lofmean, lofstd) + return nof, nod, lof, lofmin, lofmax, lofmean, lofstd def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = filter(lambda (sig, (depth, code)): depth == 1, features.iteritems()) + nof1 = filter(lambda t: t[1][0] == 1, features.items()) # t = (sig, (depth, code)) return nof1 @@ -808,13 +825,13 @@ def _compareFeatureCode(fcode): """ fcodes = set(fcode) - if (len(fcodes) == 1 and len(fcode) > 1): + if len(fcodes) == 1 and len(fcode) > 1: return "hom" - if (len(fcode) == len(fcodes)): + if len(fcode) == len(fcodes): return "het" - if (len(fcode) > len(fcodes)): + if len(fcode) > len(fcodes): return "hethom" scode = {} @@ -824,30 +841,30 @@ def _compareFeatureCode(fcode): hom = {} hethom = {} - for (key, (_, item)) in features.iteritems(): + for key, (_, item) in features.items(): # distinguish according to feature-signature # shared code - if ('||' in key and (not '&&' in key)): + if '||' in key and '&&' not in key: scode[key] = item # derivative only && - if ('&&' in key and (not '||' in key)): + if '&&' in key and '||' not in key: deriv[key] = item # combination shared code and derivative - if ('&&' in key and '||' in key): + if '&&' in key and '||' in key: desc[key] = item # distinguish according to feature-code ret = _compareFeatureCode(item) - if (ret == "het"): + if ret == "het": het[key] = item - if (ret == "hom"): + if ret == "hom": hom[key] = item - if (ret == "hethom"): + if ret == "hethom": hethom[key] = item - return (scode, deriv, desc, het, hom, hethom) + return scode, deriv, desc, het, hom, hethom def _getNumOfDefines(defset): @@ -883,27 +900,35 @@ def _getScatteringTanglingDegrees(sigs, defines): def __add(x, y): """This method is a helper function to add values of lists pairwise. See below for more information.""" - return x+y + return x + y - scat = list() # relation define to signatures - tang = [0]*len(sigs) # signatures overall + scat = list() # relation define to signatures + tang = [0] * len(sigs) # signatures overall for d in defines: - dre = re.compile(r'\b'+d+r'\b') # using word boundaries - vec = map(lambda s: not dre.search(s) is None, sigs) + dre = re.compile(r'\b' + d + r'\b') # using word boundaries + vec = list(map(lambda s: not dre.search(s) is None, sigs)) scat.append(vec.count(True)) - tang = map(__add, tang, vec) + tang = list(map(__add, tang, vec)) - if (len(scat)): sdegmean = pstat.stats.lmean(scat) - else: sdegmean = 0 - if (len(scat) > 1): sdegstd = pstat.stats.lstdev(scat) - else: sdegstd = 0 + if len(scat): + sdegmean = np.mean(scat) + else: + sdegmean = 0 + if len(scat) > 1: + sdegstd = np.std(scat) + else: + sdegstd = 0 - if (len(tang)): tdegmean = pstat.stats.lmean(tang) - else: tdegmean = 0 - if (len(tang) > 1): tdegstd = pstat.stats.lstdev(tang) - else: tdegstd = 0 + if len(tang): + tdegmean = np.mean(tang) + else: + tdegmean = 0 + if len(tang) > 1: + tdegstd = np.std(tang) + else: + tdegstd = 0 - return (sdegmean, sdegstd, tdegmean, tdegstd) + return sdegmean, sdegstd, tdegmean, tdegstd def _getGranularityStats(fcodetags): @@ -915,42 +940,42 @@ def _getGranularityStats(fcodetags): expr -> expression """ _interestingtags = [ - 'define', # define statement - 'include', # include statement - 'decl_stmt', # declaration statement - 'expr_stmt', # expression statement - 'function_decl', # function declaration - 'parameter_list', # parameter list - 'param', # parameter + 'define', # define statement + 'include', # include statement + 'decl_stmt', # declaration statement + 'expr_stmt', # expression statement + 'function_decl', # function declaration + 'parameter_list', # parameter list + 'param', # parameter ] - _curskiptag = None # holds the element we are going to skip for - _skipedtags = [] # if we find one of the elements above, we - # start skipping the following tags, until - # the corresponding endtag is found; if we - # do not find the endtag, we are going to - # start analyzing the elements in _skipedtags + _curskiptag = None # holds the element we are going to skip for + _skipedtags = [] # if we find one of the elements above, we + # start skipping the following tags, until + # the corresponding endtag is found; if we + # do not find the endtag, we are going to + # start analyzing the elements in _skipedtags granstats = dict() for tag in _interestingtags: granstats[tag] = 0 - for (_, ftagsl) in fcodetags: + for _, ftagsl in fcodetags: _curskiptag = None _skipedtags = [] - for (tag, _, sourceline) in ftagsl: - if _curskiptag == None and tag in _interestingtags: + for tag, _, sourceline in ftagsl: + if _curskiptag is None and tag in _interestingtags: _curskiptag = tag continue - if _curskiptag != None and tag == _curskiptag: + if _curskiptag is not None and tag == _curskiptag: granstats[tag] = granstats[tag] + 1 _curskiptag = None _skipedtags = [] continue - if _curskiptag != None: + if _curskiptag is not None: _skipedtags.append((tag, sourceline)) continue -# if _skipedtags != []: -# print("ERROR: ", _skipedtags) + # if _skipedtags != []: + # print("ERROR: ", _skipedtags) return granstats @@ -964,7 +989,7 @@ def _checkSigEquivalence(sig1, sig2): It uses an xmlrpc-call on troi.fim.uni-passau.de.""" global __errorfexp global __errormatch - if not (sig1 and sig2): # ommit empty signatures + if not (sig1 and sig2): # ommit empty signatures return False # TODO csp: @@ -982,16 +1007,17 @@ def prettyPrintSet(s): h = s.pop() r = str(h) - for e in s: r += ','+str(e) + for e in s: + r += ',' + str(e) return r -def resetModule() : +def resetModule(): global __macrofuncs, __defset, __defsetf - __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" - __defset = set() # macro-objects - __defsetf = dict() # macro-objects per file + __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", + # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" + __defset = set() # macro-objects + __defsetf = dict() # macro-objects per file def apply(folder): @@ -1001,23 +1027,21 @@ def apply(folder): # overall status variables resetModule() - sigmap = {} # {: []} - afeatures = {} # identified features; {: ([flag], depth, [code])} + sigmap = {} # {: []} + afeatures = {} # identified features; {: ([flag], depth, [code])} def _mergeFeatures(ffeatures): """This function merges the, with the parameter given dictionary (ffeatures) to the afeatures (overall-features).""" - for (sig, (depth, code)) in ffeatures.iteritems(): - (mal, psig) = _parseFeatureSignature(sig) + for sig, (depth, code) in ffeatures.items(): + mal, psig = _parseFeatureSignature(sig) try: sigmatch = _checkForEquivalentSig(sigmap.keys(), psig) - (tmpflag, tmpdepth, tmpcode) = \ - afeatures[sigmap[sigmatch][0]] + tmpflag, tmpdepth, tmpcode = afeatures[sigmap[sigmatch][0]] tmpdepth = min(tmpdepth, depth) tmpcode += code - afeatures[sigmap[sigmatch][0]] = \ - (tmpflag, tmpdepth, tmpcode) + afeatures[sigmap[sigmatch][0]] = (tmpflag, tmpdepth, tmpcode) sigmap[sigmatch].append(sig) except NoEquivalentSigError: # mergedfeatures get the depth of minus one @@ -1031,7 +1055,7 @@ def _mergeFeatures(ffeatures): global __curfile fcount = 0 files = returnFileNames(folder, ['.xml']) - fstats = [None]*len(__statsorder) + fstats = [None] * len(__statsorder) ftotal = len(files) # get statistics for all files; write results into csv @@ -1043,45 +1067,41 @@ def _mergeFeatures(ffeatures): tree = etree.parse(file) except etree.XMLSyntaxError: print("ERROR: cannot parse (%s). Skipping this file." % - os.path.join(folder, file)) + os.path.join(folder, file)) continue print('INFO: parsing file (%5d) of (%5d) -- (%s).' % - (fcount, ftotal, os.path.join(folder, file))) + (fcount, ftotal, os.path.join(folder, file))) root = tree.getroot() try: (features, _, featuresgrouter) = _getFeatures(root) except IfdefEndifMismatchError: print("ERROR: ifdef-endif mismatch in file (%s)" % - (os.path.join(folder, file))) + (os.path.join(folder, file))) continue _mergeFeatures(features) # filter annotations that do not have any c-code # filter annotations with less than 2 features - afeatureitems = filter(lambda (a, (f, d, c)): - c != [''], afeatures.items()) - annotations = map(lambda (a, (f, b, c)): (a, f), afeatureitems) - annotations2andmore = filter(lambda (a, f): len(f) > 1, annotations) + afeatureitems = filter(lambda t: # t = (a, (f, d, c)) + t[1][2] != [''], afeatures.items()) + annotations = list(map(lambda t: (t[0], t[1][0]), afeatureitems)) # t = (a, (f, b, c)) + annotations2andmore = list(filter(lambda t: len(t[1]) > 1, annotations)) # t = (a, f) annotationmap = dict() - for (a, f) in annotations2andmore: annotationmap[a] = f + for a, f in annotations2andmore: + annotationmap[a] = f annotations2andmore = map(lambda s: set(s), annotationmap.values()) projectpath = os.path.dirname(folder) - fd = open( - os.path.join( - projectpath, - __outputfile - ) - , 'w') + fd = open(os.path.join(projectpath, __outputfile), 'w') featurenames = reduce(set.union, annotations2andmore, set([])) for i in featurenames: fd.write(i + '\n') - for (a,f) in annotationmap.iteritems(): - fd.write(prettyPrintSet(f)+';'+a+'\n') + for a, f in annotationmap.items(): + fd.write(prettyPrintSet(f) + ';' + a + '\n') fd.close() @@ -1090,11 +1110,10 @@ def _mergeFeatures(ffeatures): def addCommandLineOptionsMain(optionparser): ''' add command line options for a direct call of this script''' - optionparser.add_argument("--folder", dest="folder", - help="input folder [default: %(default)s]", default=".") + optionparser.add_argument("--folder", dest="folder", help="input folder [default: %(default)s]", default=".") -def addCommandLineOptions(optionparser) : +def addCommandLineOptions(optionparser): # TODO implement CSP solving? # optionparser.add_argument("--csp", dest="csp", action="store_true", # default=False, help="make use of csp solver to check " \ @@ -1127,8 +1146,7 @@ def getResultsFile(): # main folder = os.path.abspath(options.folder) - if (os.path.isdir(folder)): + if os.path.isdir(folder): apply(folder) else: sys.exit(-1) - diff --git a/lib/cpplib.py b/lib/cpplib.py index e6a1cc3..e351675 100644 --- a/lib/cpplib.py +++ b/lib/cpplib.py @@ -98,7 +98,7 @@ def _rewriteTwo(param): try: rsig = expr.parseString(ifdefexp)[0] - except pypa.ParseException, e: + except pypa.ParseException as e: print('ERROR (parse): cannot parse sig (%s) -- (%s)' % (ifdefexp, e.col)) return ifdefexp diff --git a/setup.py b/setup.py index 0ea0cda..a94477c 100644 --- a/setup.py +++ b/setup.py @@ -40,10 +40,10 @@ }, install_requires=[ - 'statlib==1.2', - 'pyparsing==2.*', - 'enum34', - 'lxml>=3.4' + #'statlib==1.2', + #'pyparsing==2.*', + #'enum34', + #'lxml>=3.4' ], dependency_links=[ From 6013696bc108eb077d73d91a852dcc6ef26f522e Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 10:19:55 +0100 Subject: [PATCH 02/72] Fix analyses/discipline.py and minor things in analyses/derivative.py --- analyses/derivative.py | 28 ++-- analyses/discipline.py | 352 ++++++++++++++++++++--------------------- 2 files changed, 183 insertions(+), 197 deletions(-) diff --git a/analyses/derivative.py b/analyses/derivative.py index b1781b1..b542e10 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -111,17 +111,12 @@ def returnFileNames(folder, extfilt=['.xml']): currentfolder = wqueue[0] wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) - tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) - tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) - tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles = list(filter(lambda n: os.path.isfile(os.path.join(currentfolder, n)), foldercontent)) + tmpfiles = list(filter(lambda n: os.path.splitext(n)[1] in extfilt, tmpfiles)) + tmpfiles = list(map(lambda n: os.path.join(currentfolder, n), tmpfiles)) filesfound += tmpfiles - tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) - tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders = list(filter(lambda n: os.path.isdir(os.path.join(currentfolder, n)), foldercontent)) + tmpfolders = list(map(lambda n: os.path.join(currentfolder, n), tmpfolders)) wqueue += tmpfolders return filesfound @@ -314,7 +309,7 @@ def _countNestedIfdefs(root): if len(cnlist) > 0: nnimax = max(cnlist) - nnitmp = filter(lambda n: n > 0, cnlist) + nnitmp = list(filter(lambda n: n > 0, cnlist)) nnimean = np.mean(nnitmp) else: nnimax = 0 @@ -406,7 +401,7 @@ def _parseAndAddDefine(node): except pypa.ParseException: return - iden = ''.join(map(str, res[0])) + iden = ''.join(list(map(str, res[0]))) expn = res[-1] para = res[1:-1] __macrofuncs[iden] = (para, expn) @@ -797,7 +792,7 @@ def _getFeatureStats(features): def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = filter(lambda t: t[1][0] == 1, features.items()) # t = (sig, (depth, code)) + nof1 = list(filter(lambda t: t[1][0] == 1, features.items())) # t = (sig, (depth, code)) return nof1 @@ -880,7 +875,7 @@ def _getNumOfDefines(defset): # basic operation of this function is to check __defset against # __macrofuncs funcmacros = __macrofuncs.keys() - funcmacros = map(lambda n: n.split('(')[0], funcmacros) + funcmacros = list(map(lambda n: n.split('(')[0], funcmacros)) funcmacros = set(funcmacros) return len((defset - funcmacros)) @@ -1084,15 +1079,14 @@ def _mergeFeatures(ffeatures): # filter annotations that do not have any c-code # filter annotations with less than 2 features - afeatureitems = filter(lambda t: # t = (a, (f, d, c)) - t[1][2] != [''], afeatures.items()) + afeatureitems = list(filter(lambda t: t[1][2] != [''], afeatures.items())) # t = (a, (f, d, c)) annotations = list(map(lambda t: (t[0], t[1][0]), afeatureitems)) # t = (a, (f, b, c)) annotations2andmore = list(filter(lambda t: len(t[1]) > 1, annotations)) # t = (a, f) annotationmap = dict() for a, f in annotations2andmore: annotationmap[a] = f - annotations2andmore = map(lambda s: set(s), annotationmap.values()) + annotations2andmore = list(map(lambda s: set(s), annotationmap.values())) projectpath = os.path.dirname(folder) fd = open(os.path.join(projectpath, __outputfile), 'w') diff --git a/analyses/discipline.py b/analyses/discipline.py index c34ae63..7eff868 100755 --- a/analyses/discipline.py +++ b/analyses/discipline.py @@ -29,17 +29,16 @@ import sys from argparse import ArgumentParser, RawTextHelpFormatter - # ################################################# # external modules - # python-lxml module +# python-lxml module from lxml import etree -def returnFileNames(folder, extfilt = ['.xml']): - '''This function returns all files of the input folder - and its subfolders.''' +def returnFileNames(folder, extfilt=['.xml']): + """This function returns all files of the input folder + and its subfolders.""" filesfound = list() if os.path.isdir(folder): @@ -50,16 +49,16 @@ def returnFileNames(folder, extfilt = ['.xml']): wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) + tmpfiles) tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles) filesfound += tmpfiles tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders) wqueue += tmpfolders return filesfound @@ -72,8 +71,9 @@ class DisciplinedAnnotations: __cppnsdef = 'http://www.srcML.org/srcML/src' __cpprens = re.compile('{(.+)}(.+)') __conditionals = ['if', 'ifdef', 'ifndef', 'else', 'elif', 'endif'] - __conditions = ['if', 'ifdef', 'ifndef'] + __conditions = ['if', 'ifdef', 'ifndef'] outputfile = "cppstats_discipline.csv" + ################################################## def __init__(self, folder, options): @@ -81,10 +81,12 @@ def __init__(self, folder, options): self.opts = options self.opts.dir = os.path.abspath(folder) - print self.opts.dir + print(self.opts.dir) if not self.opts.dir: - oparser.print_help() + # TODO: oparser package is not available anymore + # oparser.print_help() + print("Fatal error") sys.exit(-1) self.overallblocks = 0 @@ -105,13 +107,13 @@ def __init__(self, folder, options): self.checkFiles() def __getIfdefAnnotations__(self, root): - '''This method returns all nodes of the xml which are ifdef - annotations in the source code.''' + """This method returns all nodes of the xml which are ifdef + annotations in the source code.""" treeifdefs = list() for _, elem in etree.iterwalk(root): - ns, tag = DisciplinedAnnotations.__cpprens.match(elem.tag).\ - groups() + ns, tag = DisciplinedAnnotations.__cpprens.match(elem.tag). \ + groups() if ns == DisciplinedAnnotations.__cppnscpp \ and tag in DisciplinedAnnotations.__conditionals: @@ -120,10 +122,11 @@ def __getIfdefAnnotations__(self, root): return treeifdefs def __createListFromTreeifdefs__(self, treeifdefs): - '''This method returns a list representation for the input treeifdefs - (xml-objects). Corresponding #ifdef elements are in one sublist.''' + """This method returns a list representation for the input treeifdefs + (xml-objects). Corresponding #ifdef elements are in one sublist.""" - if not treeifdefs: return [] + if not treeifdefs: + return [] listifdefs = list() workerlist = list() @@ -139,7 +142,7 @@ def __createListFromTreeifdefs__(self, treeifdefs): return -1 workerlist[-1].append(nifdef) last = workerlist[-1] - getpairs = zip(last,last[1:]) + getpairs = zip(last, last[1:]) map(lambda i: listifdefs.append(list(i)), getpairs) workerlist = workerlist[:-1] else: @@ -151,18 +154,18 @@ def __createListFromTreeifdefs__(self, treeifdefs): return listifdefs def __filterConditionalPreprocessorDirectives(self, listifdefs): - '''This method filters out all ifdef-endif pairs that annotate only preprocessor directives.''' + """This method filters out all ifdef-endif pairs that annotate only preprocessor directives.""" # iterate annotated blocks by determining all siblings of the #ifdef and filter out preprocessor # annotated elements - resultlist = filter(lambda (ifdef, endif): ifdef.getnext() != endif, listifdefs) + resultlist = list(filter(lambda t: t[0].getnext() != t[1], listifdefs)) # t = (ifdef, endif) print('[INFO] before after: %s <-> %s' % (str(len(listifdefs)), str(len(resultlist)))) return resultlist + PATTLS = 0 # 1 << 0 => 1 - PATTLS = 0 # 1 << 0 => 1 def __checkStrictTLSFDPattern__(self, listifdefs): - '''like sibling pattern, but only top level and statement elements are - considered disciplined''' + """like sibling pattern, but only top level and statement elements are + considered disciplined""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -170,26 +173,25 @@ def __checkStrictTLSFDPattern__(self, listifdefs): nodeifdef = listcorifdef[0] nodeifdefsibs = [sib for sib in nodeifdef.itersiblings()] - error=0 + error = 0 for corifdef in listcorifdef[1:]: - if not corifdef in nodeifdefsibs: - error=1 + if corifdef not in nodeifdefsibs: + error = 1 - if error==0: + if error == 0: parenttag = self.__getParentTag__(nodeifdef) - if not parenttag in ['block','public']: - error=1 - if error==1: + if parenttag not in ['block', 'public']: + error = 1 + if error == 1: listundisciplinedunknown.append(listcorifdef) else: listundisciplinedknown.append(listcorifdef) - return (listundisciplinedknown, listundisciplinedunknown) - + return listundisciplinedknown, listundisciplinedunknown def __checkStrictTLSCUPattern__(self, listifdefs): - '''This method checks all patterns, if they occur right under the root element - of the grammer, here unit.''' + """This method checks all patterns, if they occur right under the root element + of the grammer, here unit.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -197,26 +199,25 @@ def __checkStrictTLSCUPattern__(self, listifdefs): nodeifdef = listcorifdef[0] nodeifdefsibs = [sib for sib in nodeifdef.itersiblings()] - error=0 + error = 0 for corifdef in listcorifdef[1:]: - if not corifdef in nodeifdefsibs: - error=1 + if corifdef not in nodeifdefsibs: + error = 1 - if error==0: + if error == 0: parenttag = self.__getParentTag__(nodeifdef) - if not parenttag in ['unit']: - error=1 - if error==1: + if parenttag not in ['unit']: + error = 1 + if error == 1: listundisciplinedunknown.append(listcorifdef) else: listundisciplinedknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) - + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown def __checkStrictPattern__(self, listifdefs): - '''This pattern checks the annotation of functions, where the XML markup + """This pattern checks the annotation of functions, where the XML markup of src2srcml is ill-formed. TODO might be fixed in future versions of src2srcml. Example is: void foo(k) @@ -224,7 +225,7 @@ def __checkStrictPattern__(self, listifdefs): { // some lines of code } - ''' + """ listundisciplinedknown = list() listundisciplinedunknown = list() @@ -237,26 +238,27 @@ def __checkStrictPattern__(self, listifdefs): nodeendif = listcorifdef[1] func = nodeendif.getparent() - if func != None and func.tag.split('}')[1] == 'function': + if func is not None and func.tag.split('}')[1] == 'function': nodefuncsibs = [sib for sib in func.itersiblings(preceding=True)] if nodeifdef == nodefuncsibs[0]: if self.opts.verbose: - print('[INFO] ill-formed compilation unit pattern occured in line (%4s).' % nodeifdef.sourceline) + print( + '[INFO] ill-formed compilation unit pattern occured in line (%4s).' % nodeifdef.sourceline) listundisciplinedknown.append(listcorifdef) continue listundisciplinedunknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown + PATSIB = 1 # 1 << 1 => 2 - PATSIB = 1 # 1 << 1 => 2 def __checkSiblingPattern__(self, listifdefs): - '''This method returns a tuple with (listdisciplined, + """This method returns a tuple with (listdisciplined, listundisciplined) #ifdef elements. The pattern works on the basis of the sibling pattern. If the xml elements of #if-#elif-#else-#endif - are siblings, we determine them as disciplined.''' + are siblings, we determine them as disciplined.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -264,31 +266,30 @@ def __checkSiblingPattern__(self, listifdefs): nodeifdef = listcorifdef[0] nodeifdefsibs = [sib for sib in nodeifdef.itersiblings()] - error=0; + error = 0 for corifdef in listcorifdef[1:]: - if not corifdef in nodeifdefsibs: - error=1 - if error==1: + if corifdef not in nodeifdefsibs: + error = 1 + if error == 1: listundisciplinedunknown.append(listcorifdef) else: listundisciplinedknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) - + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown def __getParentTag__(self, tag): parent = tag.getparent() return parent.tag.split('}')[1] + PATIFTHEN = 2 # 1 << 2 => 4 - PATIFTHEN = 2 # 1 << 2 => 4 def __checkIfThenPattern__(self, listifdefs): - '''This method returns a tuple with (listdisciplined, + """This method returns a tuple with (listdisciplined, listundisciplined) #ifdef elements. The pattern matches the following situation. The if-then in C is enframed by #if-#endif. The else part of the if-then in C is not enframed. The sibling pattern does not work here - since the annatation cannot work properly here.''' + since the annatation cannot work properly here.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -318,14 +319,14 @@ def __checkIfThenPattern__(self, listifdefs): else: listundisciplinedunknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown - #TODO + # TODO def __checkForWrapperPattern__(self, listifdefs): - '''This method returns a tuple with (listdisciplined, + """This method returns a tuple with (listdisciplined, listundisciplined) #ifdef elements. The pattern matches the following - situation. The for in C is enframed by #if-#endif.''' + situation. The for in C is enframed by #if-#endif.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -355,14 +356,14 @@ def __checkForWrapperPattern__(self, listifdefs): else: listundisciplinedunknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown + PATCASE = 3 # 1 << 3 => 8 - PATCASE = 3 # 1 << 3 => 8 def __checkCasePattern__(self, listifdefs): - '''The method checks the case-block pattern; the #ifdef enframes a case block - of a switch case.''' + """The method checks the case-block pattern; the #ifdef enframes a case block + of a switch case.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -382,13 +383,14 @@ def __checkCasePattern__(self, listifdefs): else: listundisciplinedunknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown + + PATELSEIF = 4 # 1 << 4 => 16 - PATELSEIF = 4 # 1 << 4 => 16 def __checkElseIfPattern__(self, listifdefs): - '''The method check the elseif-block pattern; the #ifdef enframes an elseif - block in an if-then-else.''' + """The method check the elseif-block pattern; the #ifdef enframes an elseif + block in an if-then-else.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -405,7 +407,7 @@ def __checkElseIfPattern__(self, listifdefs): # else parent -> #ifdef nodeendif = listcorifdef[-1] thensib = nodeendif.getprevious() - if thensib == None: + if thensib is None: listundisciplinedunknown.append(listcorifdef) continue if thensib.tag.split('}')[1] not in ['then']: @@ -428,13 +430,14 @@ def __checkElseIfPattern__(self, listifdefs): else: listundisciplinedknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown + + PATPARAM = 5 # 1 << 5 => 32 - PATPARAM = 5 # 1 << 5 => 32 def __checkParameter__(self, listifdefs): - '''The method checks whether an #ifdef enframes a parameter of a function; - includes function definitions and function calls.''' + """The method checks whether an #ifdef enframes a parameter of a function; + includes function definitions and function calls.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -449,13 +452,13 @@ def __checkParameter__(self, listifdefs): error = 0 for corifdef in listcorifdef[1:]: - if not corifdef in nodeifdefsibs: + if corifdef not in nodeifdefsibs: error = 1 if error == 0: # check whether node is an argument or parameter parenttag = self.__getParentTag__(nodeifdef) - if not parenttag in ['argument_list','parameter_list']: + if parenttag not in ['argument_list', 'parameter_list']: error = 1 firstsib = nodeifdefsibs[0] @@ -468,12 +471,13 @@ def __checkParameter__(self, listifdefs): print('[INFO] param/argument pattern occured in line (%4s).' % nodeifdef.sourceline) listundisciplinedknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown + + PATEXP = 6 # 1 << 5 => 64 - PATEXP = 6 # 1 << 5 => 64 def __checkExpression__(self, listifdefs): - '''The method checks whether an #ifdef enframes an expression of a condition.''' + """The method checks whether an #ifdef enframes an expression of a condition.""" listundisciplinedknown = list() listundisciplinedunknown = list() @@ -507,99 +511,88 @@ def __checkExpression__(self, listifdefs): print('[INFO] expression pattern occured in line (%4s).' % nodeifdef.sourceline) listundisciplinedknown.append(listcorifdef) - assert len(listifdefs) == len(listundisciplinedknown)+len(listundisciplinedunknown) - return (listundisciplinedknown, listundisciplinedunknown) - + assert len(listifdefs) == len(listundisciplinedknown) + len(listundisciplinedunknown) + return listundisciplinedknown, listundisciplinedunknown def __iterateUnknownPatterns__(self, listifdefs, file): - '''This method iterates of the unknown patterns and prints out information - about the pattern: file and line.''' + """This method iterates of the unknown patterns and prints out information + about the pattern: file and line.""" for ifdef in listifdefs: if self.opts.log: - print('[INFO] Unknown pattern in file (%s) and line (%s)' % \ - (file, ifdef[0].sourceline)) + print(f'[INFO] Unknown pattern in file ({file}) and line ({ifdef[0].sourceline})') def __checkDiscipline__(self, treeifdefs, file): - '''This method checks a number of patterns in the given treeifdefs. + """This method checks a number of patterns in the given treeifdefs. The checks are in that order, that ifdef patterns not recognized - are passed to the next pattern.''' + are passed to the next pattern.""" listundisciplined = self.__createListFromTreeifdefs__(treeifdefs) listundisciplined = self.__filterConditionalPreprocessorDirectives(listundisciplined) - if (listundisciplined == -1): + if listundisciplined == -1: print('[ERROR] Too many #endifs in file (%s)' % file) return - if (listundisciplined == -2): + if listundisciplined == -2: print('[ERROR] Not enough #endifs in file (%s)' % file) return self.overallblocks += len(listundisciplined) # check TLS pattern, subset of sibling pattern - if (self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATTLS)): + if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATTLS): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkStrictTLSCUPattern__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkStrictTLSCUPattern__(listifdefs) self.compilationunit += len(listdisciplined) self.disciplined += len(listdisciplined) # checking fd pattern (part of tls) listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkStrictTLSFDPattern__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkStrictTLSFDPattern__(listifdefs) self.functiontype += len(listdisciplined) self.disciplined += len(listdisciplined) # checking ill-formed compilation unit pattern listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkStrictPattern__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkStrictPattern__(listifdefs) self.compilationunit += len(listdisciplined) self.disciplined += len(listdisciplined) # check if-then pattern - if (self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATIFTHEN)): + if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATIFTHEN): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkIfThenPattern__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkIfThenPattern__(listifdefs) self.wrapperif += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check case pattern - if (self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATCASE)): + if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATCASE): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkCasePattern__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkCasePattern__(listifdefs) self.conditionalcase += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check else-if pattern - if (self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATELSEIF)): + if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATELSEIF): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkElseIfPattern__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkElseIfPattern__(listifdefs) self.conditionalelif += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check param pattern - if (self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATPARAM)): + if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATPARAM): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkParameter__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkParameter__(listifdefs) self.parameter += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check expression pattern - if (self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATEXP)): + if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATEXP): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkExpression__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkExpression__(listifdefs) self.expression += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check sibling pattern; check this late because pattern might match for others as well - if (self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATSIB)): + if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATSIB): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = \ - self.__checkSiblingPattern__(listifdefs) + (listdisciplined, listundisciplined) = self.__checkSiblingPattern__(listifdefs) self.siblings += len(listdisciplined) self.disciplined += len(listdisciplined) @@ -616,7 +609,7 @@ def checkFile(self, file): return # get LOC - self.loc += len(f.readlines())-2; + self.loc += len(f.readlines()) - 2 # get root of the xml and iterate over it root = tree.getroot() @@ -629,7 +622,7 @@ def checkFile(self, file): def checkFiles(self): xmlfiles = returnFileNames(self.opts.dir, ['.xml']) - for xmlfile in xmlfiles: + for xmlfile in xmlfiles: print('[INFO] checking file %s' % xmlfile) self.checkFile(xmlfile) projectpath = os.path.dirname(self.opts.dir) @@ -641,70 +634,70 @@ def checkFiles(self): ), 'w') ratio = 0 - if (self.overallblocks > 0): - ratio = self.disciplined/(0.0 + self.overallblocks) + if self.overallblocks > 0: + ratio = self.disciplined / (0.0 + self.overallblocks) fd.write("projectname" - +";"+"loc" - +";"+"compilationunit" - +";"+"functiontype" - +";"+"siblings" - +";"+"wrapperif" - +";"+"conditionalcase" - +";"+"conditionalelif" - +";"+"parameter" - +";"+"expression" - +";"+"undisciplinedknown" - +";"+"undisciplinedunknown" - +";"+"disciplined/overallblocks" - +";"+"overallblocks"+"\n") + + ";" + "loc" + + ";" + "compilationunit" + + ";" + "functiontype" + + ";" + "siblings" + + ";" + "wrapperif" + + ";" + "conditionalcase" + + ";" + "conditionalelif" + + ";" + "parameter" + + ";" + "expression" + + ";" + "undisciplinedknown" + + ";" + "undisciplinedunknown" + + ";" + "disciplined/overallblocks" + + ";" + "overallblocks" + "\n") fd.write(projectname - +";"+str(self.loc) - +";"+str(self.compilationunit) - +";"+str(self.functiontype) - +";"+str(self.siblings) - +";"+str(self.wrapperif) - +";"+str(self.conditionalcase) - +";"+str(self.conditionalelif) - +";"+str(self.parameter) - +";"+str(self.expression) - +";"+str(self.undisciplinedknown) - +";"+str(self.undisciplinedunknown) - +";"+str(ratio) - +";"+str(self.overallblocks)+"\n") + + ";" + str(self.loc) + + ";" + str(self.compilationunit) + + ";" + str(self.functiontype) + + ";" + str(self.siblings) + + ";" + str(self.wrapperif) + + ";" + str(self.conditionalcase) + + ";" + str(self.conditionalelif) + + ";" + str(self.parameter) + + ";" + str(self.expression) + + ";" + str(self.undisciplinedknown) + + ";" + str(self.undisciplinedunknown) + + ";" + str(ratio) + + ";" + str(self.overallblocks) + "\n") # ################################################## # add command line options def addCommandLineOptionsMain(optionparser): - ''' add command line options for a direct call of this script''' + """ add command line options for a direct call of this script""" optionparser.add_argument('-d', '--dir', '--folder', dest='dir', - help='input directory (mandatory)') + help='input directory (mandatory)') -def addCommandLineOptions(optionparser) : +def addCommandLineOptions(optionparser): optionparser.description = 'This analysis counts the number of the disciplined CPP usage in software projects. \n' \ - 'To this end, it checks xml representations of header and source ' \ - 'files and returns the number of disciplined ifdefs in those. \n' - # TODO what are the disciplined ones - #'Disciplined annotations are:' + 'To this end, it checks xml representations of header and source ' \ + 'files and returns the number of disciplined ifdefs in those. \n' + # TODO what are the disciplined ones + # 'Disciplined annotations are:' optionparser.add_argument('-l', '--log', dest='log', action="store_true", - default=True, help='log to stdout [default=%(default)s]') + default=True, help='log to stdout [default=%(default)s]') optionparser.add_argument('-v', '--verbose', dest='verbose', action="store_true", - default=False, help='verbose output [default=%(default)s]') + default=False, help='verbose output [default=%(default)s]') optionparser.add_argument('--check', dest='check', type=int, - default=1, help='CHECK sets the patterns that are checked [default=%(default)s].\n' - 'Supply sum of wanted patterns:\n' - '(1) check top level siblings (compilation unit) \n' - '(2) check sibling (excludes check top level siblings; NOT CLASSIFIED) \n' - '(4) check if-then enframement (wrapper) \n' - '(8) check case enframement (conditional) \n' - '(16) check else-if enframement (conditional) \n' - '(32) check param/argument enframement (parameter) \n' - '(64) check expression enframement (expression) \n' - '(128) check else enframement (NOT CLASSIFIED) \n' - ) + default=1, help='CHECK sets the patterns that are checked [default=%(default)s].\n' + 'Supply sum of wanted patterns:\n' + '(1) check top level siblings (compilation unit) \n' + '(2) check sibling (excludes check top level siblings; NOT CLASSIFIED) \n' + '(4) check if-then enframement (wrapper) \n' + '(8) check case enframement (conditional) \n' + '(16) check else-if enframement (conditional) \n' + '(32) check param/argument enframement (parameter) \n' + '(64) check expression enframement (expression) \n' + '(128) check else enframement (NOT CLASSIFIED) \n' + ) optionparser.add_argument('--dall', dest='disc_all', action="store_true", default=True, help='check all patterns [default=%(default)s] \n(overrides --check)') @@ -718,7 +711,6 @@ def getResultsFile(): ################################################## if __name__ == '__main__': - ################################################## # options parsing parser = ArgumentParser(formatter_class=RawTextHelpFormatter) From 55c81f859a615a68c1455892f4034a54e5910a91 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 10:27:27 +0100 Subject: [PATCH 03/72] Fix analyses/featurelocation --- analyses/featurelocations.py | 165 ++++++++++++++++------------------- 1 file changed, 73 insertions(+), 92 deletions(-) diff --git a/analyses/featurelocations.py b/analyses/featurelocations.py index 0f49927..6fa02a0 100644 --- a/analyses/featurelocations.py +++ b/analyses/featurelocations.py @@ -27,36 +27,32 @@ import os import re import sys -import xmlrpclib from argparse import ArgumentParser, RawTextHelpFormatter - # ################################################# # path adjustments, so that all imports can be done relative to these paths __lib_subfolder = "lib" sys.path.append(os.path.abspath(__lib_subfolder)) # lib subfolder - # ################################################# # external modules # enums from enum import Enum - # python-lxml module +# python-lxml module from lxml import etree # pyparsing module import pyparsing as pypa -pypa.ParserElement.enablePackrat() # speed up parsing -sys.setrecursionlimit(8000) # handle larger expressions +pypa.ParserElement.enablePackrat() # speed up parsing +sys.setrecursionlimit(8000) # handle larger expressions # ################################################# # config: __outputfile = "cppstats_featurelocations.csv" __listoffeaturesfile = "listoffeatures.csv" - # ################################################# # constants: @@ -79,15 +75,16 @@ __defset = set() # macro-objects __defsetf = dict() # macro-objects per file + # collected statistics class __statsorder(Enum): - FILENAME = 0 # name of the file + FILENAME = 0 # name of the file LINE_START = 1 # starting line of an #ifdef block - LINE_END = 2 # ending line of an #ifdef block (ends either at #else, - # #elif, or #endif on same level) - TYPE = 3 # either #if, #elif, or #else + LINE_END = 2 # ending line of an #ifdef block (ends either at #else, + # #elif, or #endif on same level) + TYPE = 3 # either #if, #elif, or #else EXPRESSION = 4 # the presence condition stated in the #ifdef - CONSTANTS = 5 # all configuration constants used in the presence condition + CONSTANTS = 5 # all configuration constants used in the presence condition ################################################## @@ -104,7 +101,7 @@ def __init__(self, filename, startline, endline, type, expression): namespace = '{' + _cppnscpp + '}' typeWithoutNamespace = '#' + type.replace(namespace, "") - self.filename = filename # TODO use relative paths here! + self.filename = filename # TODO use relative paths here! self.startline = startline self.endline = endline self.type = typeWithoutNamespace @@ -130,7 +127,7 @@ def __eq__(self, other): return False def __ne__(self, other): - return (not self.__eq__(other)) + return not self.__eq__(other) def getCSVList(self): returnList = [] @@ -190,10 +187,10 @@ def _collectDefines(d): but not #define GLIBCVER(x,y,z) ... """ __defset.add(d[0]) - if __defsetf.has_key(__curfile): + if __curfile in __defsetf: __defsetf[__curfile].add(d[0]) else: - __defsetf[__curfile] = set([d[0]]) + __defsetf[__curfile] = {d[0]} return d @@ -210,8 +207,7 @@ def _collectDefines(d): __hexadec = \ pypa.Literal('0x').suppress() + \ - pypa.Word(pypa.hexnums). \ - setParseAction(lambda t: str(int(t[0], 16))) + \ + pypa.Word(pypa.hexnums).setParseAction(lambda t: str(int(t[0], 16))) + \ pypa.Optional(__numlitu) + \ pypa.Optional(__numlitl) + \ pypa.Optional(__numlitl) @@ -223,14 +219,11 @@ def _collectDefines(d): pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + \ pypa.Optional(pypa.Suppress(pypa.Literal('L'))) -__identifier = \ - pypa.Word(pypa.alphanums + '_' + '-' + '@' + '$').setParseAction(_collectDefines) +__identifier = pypa.Word(pypa.alphanums + '_' + '-' + '@' + '$').setParseAction(_collectDefines) __arg = pypa.Word(pypa.alphanums + '_') -__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + \ - __arg) +__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + __arg) __fname = pypa.Word(pypa.alphas, pypa.alphanums + '_') -__function = pypa.Group(__fname + pypa.Literal('(').suppress() + \ - __args + pypa.Literal(')').suppress()) +__function = pypa.Group(__fname + pypa.Literal('(').suppress() + __args + pypa.Literal(')').suppress()) class NoEquivalentSigError(Exception): @@ -238,12 +231,12 @@ def __init__(self): pass def __str__(self): - return ("No equivalent signature found!") + return "No equivalent signature found!" class IfdefEndifMismatchError(Exception): def __str__(self): - return ("Ifdef and endif do not match!") + return "Ifdef and endif do not match!" def _parseFeatureSignatureAndRewrite(sig): @@ -258,7 +251,7 @@ def _parseFeatureSignatureAndRewrite(sig): # if no equivalence can be found a name rewriting is done # e.g. 'defined' __pt = { - #'defined' : 'defined_', + # 'defined' : 'defined_', 'defined': '', '!': '¬', '&&': '&and', @@ -289,7 +282,6 @@ def _rewriteOne(param): ret = __pt[param[0][0]] + str(param[0][1]) return ret - def _rewriteTwo(param): """This function returns each two parameter function representation for maple.""" @@ -304,8 +296,7 @@ def _rewriteTwo(param): ret = '(true &and ' + ret + ')' return ret - operand = __string | __hexadec | __integer | \ - __function | __identifier + operand = __string | __hexadec | __integer | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') expr = pypa.operatorPrecedence(operand, [ @@ -319,14 +310,14 @@ def _rewriteTwo(param): try: rsig = expr.parseString(sig)[0] - except pypa.ParseException, e: + except pypa.ParseException as e: print('ERROR (parse): cannot parse sig (%s) -- (%s)' % (sig, e.col)) return sig except RuntimeError: - print('ERROR (time): cannot parse sig (%s)' % (sig)) + print('ERROR (time): cannot parse sig (%s)' % sig) return sig - except ValueError, e: + except ValueError as e: print('ERROR (parse): cannot parse sig (%s) ~~ (%s)' % (sig, e)) return sig @@ -385,9 +376,9 @@ def _getMacroSignature(ifdefnode): # get either the expr or the name tag, # which is always the second descendant - if (tag in ['if', 'elif', 'ifdef', 'ifndef']): + if tag in ['if', 'elif', 'ifdef', 'ifndef']: nexpr = [itex for itex in ifdefnode.iterdescendants()] - if (len(nexpr) == 1): + if len(nexpr) == 1: res = nexpr[0].tail else: nexpr = nexpr[1] @@ -451,7 +442,7 @@ def _wrapGrOuterUp(fouter, featuresgrouter, eelem): itouter = fouter[-1] # feature surround tags fouter = fouter[:-1] - for i in xrange(0, len(itouter)): + for i in range(0, len(itouter)): sig = itouter[i][0] elem = itouter[i][1] @@ -460,12 +451,11 @@ def _wrapGrOuterUp(fouter, featuresgrouter, eelem): end = itouter[i + 1][1] if (i < len(itouter) - 1) else eelem featuresgrouter.append((sig, elem, end)) - return (fouter, featuresgrouter) - + return fouter, featuresgrouter def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # wrap up the feature - if (not flist): + if not flist: raise IfdefEndifMismatchError() itsig = flist[-1] # feature signature flist = flist[:-1] @@ -477,14 +467,14 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): finner = finner[:-1] # handle the feature code - if (features.has_key(itsig)): + if itsig in features: features[itsig][1].append(itcode) else: features[itsig] = (len(flist) + 1, [itcode]) # handle the inner granularity featuresgrinner.append((itsig, itinner)) - return (features, featuresgrinner, fcode, flist, finner) + return features, featuresgrinner, fcode, flist, finner features = {} # see above; return value featuresgrinner = [] # see above; return value @@ -514,20 +504,20 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # handling conditionals # hitting on conditional-macro if ((tag in __conditionals_all) - and (event == 'start') - and (ns == _cppnscpp)): # check the cpp:namespace + and (event == 'start') + and (ns == _cppnscpp)): # check the cpp:namespace parcon = True # hitting next conditional macro; any of ifdef, else or elif if ((tag in __conditionals_all) - and (event == 'end') - and (ns == _cppnscpp)): # check the cpp:namespace + and (event == 'end') + and (ns == _cppnscpp)): # check the cpp:namespace parcon = False # with else or elif we finish up the last if, therefor # we can applicate the wrapup if ((tag in __conditionals_else) - or (tag in __conditionals_elif)): + or (tag in __conditionals_elif)): (features, featuresgrinner, fcode, flist, finner) = _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner) @@ -539,7 +529,8 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): condinhist.append((tag, '')) fsig = _getFeatureSignature(condinhist) - if (tag in __conditionals): fouter.append([]) + if tag in __conditionals: + fouter.append([]) fouter[-1] += ([(fsig, elem)]) flist.append(fsig) fcode.append('') @@ -547,14 +538,12 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # hitting end-tag of elif-macro if ((tag in __conditionals_elif) - and (event == 'end') - and (ns == _cppnscpp)): + and (event == 'end') + and (ns == _cppnscpp)): parcon = False # hitting end-tag of define-macro - if ((tag in __macro_define) \ - and (event == 'end') \ - and (ns == _cppnscpp)): + if (tag in __macro_define) and (event == 'end') and (ns == _cppnscpp): _parseAndAddDefine(elem) # iterateting in subtree of conditional-node @@ -563,22 +552,16 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # handling endif-macro # hitting an endif-macro start-tag - if ((tag in __conditionals_endif) \ - and (event == "start") \ - and (ns == _cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "start") and (ns == _cppnscpp): # check the cpp:namespace parend = True # hitting the endif-macro end-tag - if ((tag in __conditionals_endif) \ - and (event == "end") \ - and (ns == _cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "end") and (ns == _cppnscpp): # check the cpp:namespace parend = False (features, featuresgrinner, fcode, flist, finner) = \ - _wrapFeatureUp(features, featuresgrinner, - fcode, flist, finner) - (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, - featuresgrouter, elem) + _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner) + (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, featuresgrouter, elem) # transform feature locations and append them to global list for (asig, aselem, aeelem) in featuresgrouter: @@ -586,7 +569,7 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): aselem.tag, asig) featlocations.add(floc) - while (condinhist[-1][0] != 'if'): + while condinhist[-1][0] != 'if': condinhist = condinhist[:-1] condinhist = condinhist[:-1] @@ -595,19 +578,19 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): continue # collect the source-code of the feature - if (len(flist)): - if ((event == "start") and (elem.text)): + if len(flist): + if (event == "start") and elem.text: fcode[-1] += elem.text - if ((event == "end") and (elem.tail)): + if (event == "end") and elem.tail: fcode[-1] += elem.tail - if (ns == __cppnsdef or tag not in __conditionals_all): + if ns == __cppnsdef or tag not in __conditionals_all: finner[-1].append((tag, event, elem.sourceline)) - if (flist): + if flist: raise IfdefEndifMismatchError() - return (features, featuresgrinner, featuresgrouter) + return features, featuresgrinner, featuresgrouter def _getFeaturesAtLocations(flocations, defines): @@ -620,7 +603,7 @@ def _getFeaturesAtLocations(flocations, defines): vec = map(lambda s: not dre.search(s) is None, sigs) for floc, contained in zip(flocations, vec): - if (contained): + if contained: floc.constants.add(d) @@ -635,18 +618,18 @@ def _prologCSV(folder, file, headings): fdcsv = csv.writer(fd, delimiter=',') fdcsv.writerow(["sep=,"]) fdcsv.writerow(headings) - return (fd, fdcsv) + return fd, fdcsv ################################################## # main method -def resetModule() : +def resetModule(): global __macrofuncs, __defset, __defsetf - __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" - __defset = set() # macro-objects - __defsetf = dict() # macro-objects per file + __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", + # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" + __defset = set() # macro-objects + __defsetf = dict() # macro-objects per file def apply(folder, options): @@ -663,7 +646,7 @@ def apply(folder, options): # list-of-features file loffheadings = ['FILENAME', 'CONSTANTS'] - loffrow = [None]*len(loffheadings) + loffrow = [None] * len(loffheadings) loffhandle, loffwriter = _prologCSV(os.path.join(folder, os.pardir), __listoffeaturesfile, loffheadings) # preparations for file-loop @@ -673,7 +656,7 @@ def apply(folder, options): fcount = 0 ftotal = len(files) - #TODO rewrite comment! get statistics for all files; write results into csv + # TODO rewrite comment! get statistics for all files; write results into csv # and merge the features for file in files: __curfile = file @@ -701,10 +684,9 @@ def apply(folder, options): # print features for this file to list-of-features file featureslist = list(__defsetf[__curfile]) \ - if __defsetf.has_key(__curfile) else '' # list of features within the current file - listoffeaturesstring = ';'.join(sorted(featureslist)) # sort and join - loffwriter.writerow([__curfile, listoffeaturesstring]) # write row to file - + if __curfile in __defsetf else '' # list of features within the current file + listoffeaturesstring = ';'.join(sorted(featureslist)) # sort and join + loffwriter.writerow([__curfile, listoffeaturesstring]) # write row to file # collect feature locations and consisting used features featurelocations = list(featlocations) @@ -714,13 +696,13 @@ def apply(folder, options): # print each feature location as one line into output file for floc in featurelocations: - #adjust file name if wanted - if options.filenamesRelative : # relative file name (root is project folder (not included in path)) + # adjust file name if wanted + if options.filenamesRelative: # relative file name (root is project folder (not included in path)) floc.filename = os.path.relpath(floc.filename, folder) - if options.filenames == options.FILENAME_SRCML : # cppstats file names - pass # nothing to do here, as the file path is the cppstats path by default - if options.filenames == options.FILENAME_SOURCE : # source file name + if options.filenames == options.FILENAME_SRCML: # cppstats file names + pass # nothing to do here, as the file path is the cppstats path by default + if options.filenames == options.FILENAME_SOURCE: # source file name floc.filename = floc.filename.replace(".xml", "").replace("/_cppstats/", "/source/", 1) # print floc information to CSV file @@ -728,9 +710,8 @@ def apply(folder, options): fdcsv.writerow(row) # close output files - fd.close() # __outputfile - loffhandle.close() # __listoffeaturesfile - + fd.close() # __outputfile + loffhandle.close() # __listoffeaturesfile # ################################################## @@ -739,7 +720,7 @@ def apply(folder, options): def addCommandLineOptionsMain(optionparser): ''' add command line options for a direct call of this script''' optionparser.add_argument("--folder", dest="folder", - help="input folder [default=.]", default=".") + help="input folder [default=.]", default=".") def addCommandLineOptions(optionparser): @@ -766,7 +747,7 @@ def getResultsFile(): folder = os.path.abspath(options.folder) - if (os.path.isdir(folder)): + if os.path.isdir(folder): apply(folder) else: sys.exit(-1) From 70884b1d7a69c79a6246697117bf334a4cd39d59 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 10:51:00 +0100 Subject: [PATCH 04/72] Fix analyses/general.py --- analyses/general.py | 790 ++++++++++++++++++++++---------------------- 1 file changed, 392 insertions(+), 398 deletions(-) diff --git a/analyses/general.py b/analyses/general.py index de36c18..f34bd68 100644 --- a/analyses/general.py +++ b/analyses/general.py @@ -28,31 +28,30 @@ import os import re import sys -import xmlrpclib +import numpy as np +from functools import reduce from argparse import ArgumentParser, RawTextHelpFormatter - # ################################################# # path adjustments, so that all imports can be done relative to these paths __lib_subfolder = "lib" sys.path.append(os.path.abspath(__lib_subfolder)) # lib subfolder - # ################################################# # external modules # enums from enum import Enum - # python-lxml module +# python-lxml module from lxml import etree - # statistics module -from statlib import pstat +# statistics module +# from statlib import pstat # pyparsing module import pyparsing as pypa -pypa.ParserElement.enablePackrat() # speed up parsing -sys.setrecursionlimit(8000) # handle larger expressions +pypa.ParserElement.enablePackrat() # speed up parsing +sys.setrecursionlimit(8000) # handle larger expressions ################################################## # config: @@ -78,52 +77,53 @@ __conditionals_else = ['else'] __conditionals_endif = ['endif'] __conditionals_all = __conditionals + __conditionals_elif + \ - __conditionals_else + __conditionals_else __macro_define = ['define'] -__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" -__curfile = '' # current processed xml-file -__defset = set() # macro-objects -__defsetf = dict() # macro-objects per file +__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", +# used as "GLIBVERSION(x,y,z) 100*x+10*y+z" +__curfile = '' # current processed xml-file +__defset = set() # macro-objects +__defsetf = dict() # macro-objects per file + # collected statistics class __statsorder(Enum): - FILENAME = 0 # name of the file - LOC = 1 # lines of code - NOFC = 2 # number of feature constants - LOF = 3 # number of feature code lines - ANDAVG = 4 # average nested ifdefs depth - ANDSTDEV = 5 # standard deviation for ifdefs - SDEGMEAN = 6 # shared code degree: mean - SDEGSTD = 7 # shared code degree: standard-deviation - TDEGMEAN = 8 # tangled code degree: mean - TDEGSTD = 9 # tangled code degree: standard-deviation + FILENAME = 0 # name of the file + LOC = 1 # lines of code + NOFC = 2 # number of feature constants + LOF = 3 # number of feature code lines + ANDAVG = 4 # average nested ifdefs depth + ANDSTDEV = 5 # standard deviation for ifdefs + SDEGMEAN = 6 # shared code degree: mean + SDEGSTD = 7 # shared code degree: standard-deviation + TDEGMEAN = 8 # tangled code degree: mean + TDEGSTD = 9 # tangled code degree: standard-deviation # type metrics - HOM = 10 # homogenous features - HET = 11 # heterogenous features - HOHE = 12 # combination of het and hom features + HOM = 10 # homogenous features + HET = 11 # heterogenous features + HOHE = 12 # combination of het and hom features # gran metrics - GRANGL = 13 # global level (compilation unit) - GRANFL = 14 # function and type level - GRANBL = 15 # if/while/for/do block extension - GRANSL = 16 # statement extension - includes string concat - GRANEL = 17 # condition block extension - includes return - GRANML = 18 # function parameter extension - GRANERR = 19 # not determined granularity - - NDMAX = 20 # maximum nesting depth in a file - NOFPFCMEAN = 21 # average number of files per feature constant - NOFPFCSTD = 22 # standard deviation for same data as for NOFPFCMEAN + GRANGL = 13 # global level (compilation unit) + GRANFL = 14 # function and type level + GRANBL = 15 # if/while/for/do block extension + GRANSL = 16 # statement extension - includes string concat + GRANEL = 17 # condition block extension - includes return + GRANML = 18 # function parameter extension + GRANERR = 19 # not determined granularity -################################################## + NDMAX = 20 # maximum nesting depth in a file + NOFPFCMEAN = 21 # average number of files per feature constant + NOFPFCSTD = 22 # standard deviation for same data as for NOFPFCMEAN +################################################## + ################################################## # helper functions, constants and errors -def returnFileNames(folder, extfilt = ['.xml']): - '''This function returns all files of the input folder - and its subfolders.''' +def returnFileNames(folder, extfilt=['.xml']): + """This function returns all files of the input folder + and its subfolders.""" filesfound = list() if os.path.isdir(folder): @@ -133,21 +133,17 @@ def returnFileNames(folder, extfilt = ['.xml']): currentfolder = wqueue[0] wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) - tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) - tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) - tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles = list(filter(lambda n: os.path.isfile(os.path.join(currentfolder, n)), foldercontent)) + tmpfiles = list(filter(lambda n: os.path.splitext(n)[1] in extfilt, tmpfiles)) + tmpfiles = list(map(lambda n: os.path.join(currentfolder, n), tmpfiles)) filesfound += tmpfiles - tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) - tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders = list(filter(lambda n: os.path.isdir(os.path.join(currentfolder, n)), foldercontent)) + tmpfolders = list(map(lambda n: os.path.join(currentfolder, n), tmpfolders)) wqueue += tmpfolders return filesfound + def _flatten(l): """This function takes a list as input and returns a flatten version of the list. So all nested lists are unpacked and moved up to the @@ -160,7 +156,7 @@ def _flatten(l): i -= 1 break else: - l[i:i+1] = l[i] + l[i:i + 1] = l[i] i += 1 return l @@ -170,7 +166,7 @@ def dictinvert(d): values into a dictionary that maps the values to the corresponding set of former keys.""" inv = dict() - for (k,v) in d.iteritems(): + for (k, v) in d.iteritems(): for value in v: keys = inv.setdefault(value, []) keys.append(k) @@ -184,10 +180,10 @@ def _collectDefines(d): but not #define GLIBCVER(x,y,z) ... """ __defset.add(d[0]) - if __defsetf.has_key(__curfile): + if __curfile in __defsetf: __defsetf[__curfile].add(d[0]) else: - __defsetf[__curfile] = set([d[0]]) + __defsetf[__curfile] = {d[0]} return d @@ -203,41 +199,42 @@ def _collectDefines(d): __string = pypa.QuotedString('\'', '\\') __hexadec = \ - pypa.Literal('0x').suppress() + \ - pypa.Word(pypa.hexnums).\ - setParseAction(lambda t: str(int(t[0], 16))) + \ - pypa.Optional(__numlitu) + \ - pypa.Optional(__numlitl) + \ - pypa.Optional(__numlitl) + pypa.Literal('0x').suppress() + \ + pypa.Word(pypa.hexnums).setParseAction(lambda t: str(int(t[0], 16))) + \ + pypa.Optional(__numlitu) + \ + pypa.Optional(__numlitl) + \ + pypa.Optional(__numlitl) __integer = \ - pypa.Optional('~') + \ - pypa.Word(pypa.nums+'-').setParseAction(lambda t: str(int(t[0]))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('U'))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + pypa.Optional('~') + \ + pypa.Word(pypa.nums + '-').setParseAction(lambda t: str(int(t[0]))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('U'))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('L'))) __identifier = \ - pypa.Word(pypa.alphanums+'_'+'-'+'@'+'$').setParseAction(_collectDefines) -__arg = pypa.Word(pypa.alphanums+'_') -__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + \ - __arg) + pypa.Word(pypa.alphanums + '_' + '-' + '@' + '$').setParseAction(_collectDefines) +__arg = pypa.Word(pypa.alphanums + '_') +__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + __arg) __fname = pypa.Word(pypa.alphas, pypa.alphanums + '_') -__function = pypa.Group(__fname + pypa.Literal('(').suppress() + \ - __args + pypa.Literal(')').suppress()) +__function = pypa.Group(__fname + pypa.Literal('(').suppress() + __args + pypa.Literal(')').suppress()) class NoEquivalentSigError(Exception): def __init__(self): pass + def __str__(self): - return ("No equivalent signature found!") + return "No equivalent signature found!" + class IfdefEndifMismatchError(Exception): def __init__(self): pass + def __str__(self): - return ("Ifdef and endif do not match!") + return "Ifdef and endif do not match!" + ################################################## @@ -258,24 +255,24 @@ def _parseFeatureSignatureAndRewriteCSP(sig): This one is used to make use of a csp solver without using a predicate.""" __pt = { - #'defined' : 'defined_', - 'defined' : '', - '!' : '!', + # 'defined' : 'defined_', + 'defined': '', + '!': '!', '&&': '&', '||': '|', - '<' : '_lt_', - '>' : '_gt_', + '<': '_lt_', + '>': '_gt_', '<=': '_le_', '>=': '_ge_', '==': '_eq_', '!=': '_ne_', - '*' : '_mu_', - '/' : '_di_', - '%' : '_mo_', - '+' : '_pl_', - '-' : '_mi_', - '&' : '_ba_', - '|' : '_bo_', + '*': '_mu_', + '/': '_di_', + '%': '_mo_', + '+': '_pl_', + '-': '_mi_', + '&': '_ba_', + '|': '_bo_', '>>': '_sr_', '<<': '_sl_', } @@ -286,25 +283,26 @@ def _rewriteOne(param): representation for csp.""" op, ma = param[0] mal.append(ma) - if op == '!': ret = __pt[op] + '(' + ma + ')' - if op == 'defined': ret = ma - return ret + if op == '!': + ret = __pt[op] + '(' + ma + ')' + if op == 'defined': + ret = ma + return ret def _rewriteTwo(param): """This function returns each two parameter function representation for csp.""" mal.extend(param[0][0::2]) ret = __pt[param[0][1]] - ret = '(' + ret.join(map(str, param[0][0::2])) + ')' + ret = '(' + ret.join(list(map(str, param[0][0::2]))) + ')' return ret - operand = __hexadec | __integer | __string | \ - __function | __identifier + operand = __hexadec | __integer | __string | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') expr = pypa.operatorPrecedence(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), - ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), + ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), (compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), ('&&', 2, pypa.opAssoc.LEFT, _rewriteTwo), @@ -313,14 +311,13 @@ def _rewriteTwo(param): try: rsig = expr.parseString(sig)[0] - except pypa.ParseException, e: - print('ERROR (parse): cannot parse sig (%s) -- (%s)' % - (sig, e.col)) + except pypa.ParseException as e: + print(f'ERROR (parse): cannot parse sig ({sig}) -- ({e.col})') return sig except RuntimeError: - print('ERROR (time): cannot parse sig (%s)' % (sig)) + print(f'ERROR (time): cannot parse sig ({sig})') return sig - return (mal, ''.join(rsig)) + return mal, ''.join(rsig) def _parseFeatureSignatureAndRewrite(sig): @@ -335,26 +332,26 @@ def _parseFeatureSignatureAndRewrite(sig): # if no equivalence can be found a name rewriting is done # e.g. 'defined' __pt = { - #'defined' : 'defined_', - 'defined' : '', - '!' : '¬', + # 'defined' : 'defined_', + 'defined': '', + '!': '¬', '&&': '&and', '||': '&or', - '<' : '<', - '>' : '>', + '<': '<', + '>': '>', '<=': '<=', '>=': '>=', '==': '=', '!=': '!=', - '*' : '*', # needs rewriting with parenthesis - '/' : '/', - '%' : '', # needs rewriting a % b => modp(a, b) - '+' : '+', - '-' : '-', - '&' : '', # needs rewriting a & b => BitAnd(a, b) - '|' : '', # needs rewriting a | b => BitOr(a, b) - '>>': '>>', # needs rewriting a >> b => a / (2^b) - '<<': '<<', # needs rewriting a << b => a * (2^b) + '*': '*', # needs rewriting with parenthesis + '/': '/', + '%': '', # needs rewriting a % b => modp(a, b) + '+': '+', + '-': '-', + '&': '', # needs rewriting a & b => BitAnd(a, b) + '|': '', # needs rewriting a | b => BitOr(a, b) + '>>': '>>', # needs rewriting a >> b => a / (2^b) + '<<': '<<', # needs rewriting a << b => a * (2^b) } def _rewriteOne(param): @@ -364,8 +361,7 @@ def _rewriteOne(param): ret = __pt[param[0][0]] + '(' + str(param[0][1]) + ')' if param[0][0] == 'defined': ret = __pt[param[0][0]] + str(param[0][1]) - return ret - + return ret def _rewriteTwo(param): """This function returns each two parameter function @@ -375,19 +371,18 @@ def _rewriteTwo(param): return 'modp(' + param[0][0] + ',' + param[0][2] + ')' ret = ' ' + __pt[param[0][1]] + ' ' - ret = '(' + ret.join(map(str, param[0][0::2])) + ')' + ret = '(' + ret.join(list(map(str, param[0][0::2]))) + ')' if param[0][1] in ['<', '>', '<=', '>=', '!=', '==']: ret = '(true &and ' + ret + ')' return ret - operand = __string | __hexadec | __integer | \ - __function | __identifier + operand = __string | __hexadec | __integer | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') expr = pypa.operatorPrecedence(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), - ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), + ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), (compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), ('&&', 2, pypa.opAssoc.LEFT, _rewriteTwo), @@ -396,16 +391,14 @@ def _rewriteTwo(param): try: rsig = expr.parseString(sig)[0] - except pypa.ParseException, e: - print('ERROR (parse): cannot parse sig (%s) -- (%s)' % - (sig, e.col)) + except pypa.ParseException as e: + print(f'ERROR (parse): cannot parse sig ({sig}) -- ({e.col})') return sig except RuntimeError: - print('ERROR (time): cannot parse sig (%s)' % (sig)) + print(f'ERROR (time): cannot parse sig ({sig})') return sig - except ValueError, e: - print('ERROR (parse): cannot parse sig (%s) ~~ (%s)' % - (sig, e)) + except ValueError as e: + print(f'ERROR (parse): cannot parse sig ({sig}) ~~ ({e})') return sig return ''.join(rsig) @@ -426,9 +419,9 @@ def _getMacroSignature(ifdefnode): # get either the expr or the name tag, # which is always the second descendant - if (tag in ['if', 'elif', 'ifdef', 'ifndef']): + if tag in ['if', 'elif', 'ifdef', 'ifndef']: nexpr = [itex for itex in ifdefnode.iterdescendants()] - if (len(nexpr) == 1): + if len(nexpr) == 1: res = nexpr[0].tail else: nexpr = nexpr[1] @@ -436,17 +429,19 @@ def _getMacroSignature(ifdefnode): return res -def _prologCSV(folder, file, headings, delimiter = ","): +def _prologCSV(folder, file, headings, delimiter=","): """prolog of the CSV-output file no corresponding _epilogCSV.""" fd = open(os.path.join(folder, file), 'w') fdcsv = csv.writer(fd, delimiter=delimiter) fdcsv.writerow(["sep=" + delimiter]) fdcsv.writerow(headings) - return (fd, fdcsv) + return fd, fdcsv __nestedIfdefsLevels = [] + + def _countNestedIfdefs(root): """This function counts the number of nested ifdefs (conditionals) within the source-file.""" @@ -456,24 +451,25 @@ def _countNestedIfdefs(root): for elem in elements: ns, tag = __cpprens.match(elem.tag).groups() - if ((tag in __conditionals_endif) - and (ns == __cppnscpp)): cncur -= 1 - if ((tag in __conditionals) - and (ns == __cppnscpp)): + if (tag in __conditionals_endif) and (ns == __cppnscpp): + cncur -= 1 + if (tag in __conditionals) and (ns == __cppnscpp): cncur += 1 cnlist.append(cncur) - if (len(cnlist) > 0): + if len(cnlist) > 0: nnimax = max(cnlist) - nnitmp = filter(lambda n: n > 0, cnlist) + nnitmp = list(filter(lambda n: n > 0, cnlist)) __nestedIfdefsLevels.append(nnitmp) - nnimean = pstat.stats.lmean(nnitmp) + nnimean = np.mean(nnitmp) else: nnimax = 0 nnimean = 0 - if (len(cnlist) > 1): nnistd = pstat.stats.lstdev(cnlist) - else: nnistd = 0 - return (nnimax, nnimean, nnistd) + if len(cnlist) > 1: + nnistd = np.std(cnlist) + else: + nnistd = 0 + return nnimax, nnimean, nnistd def _getFeatureSignature(condinhist, options): @@ -483,7 +479,7 @@ def _getFeatureSignature(condinhist, options): # signature; reason is elements like else or elif, which mean # basically invert the fname found before # rewritelist = [(tag, fname, )] - rewritelist = [None]*len(condinhist) + rewritelist = [None] * len(condinhist) cur = -1 for tag, fname in condinhist: @@ -491,8 +487,8 @@ def _getFeatureSignature(condinhist, options): if tag == 'if': rewritelist[cur] = (tag, fname, False) if tag in ['elif', 'else']: - (t, f, _) = rewritelist[cur-1] - rewritelist[cur-1] = (t, f, True) + (t, f, _) = rewritelist[cur - 1] + rewritelist[cur - 1] = (t, f, True) rewritelist[cur] = (tag, fname, False) fsig = '' @@ -530,7 +526,7 @@ def _getASTFuture(node): node itself.""" dess = [] - while (node is not None): + while node is not None: dess += [sib for sib in node.itersiblings(preceding=False)] node = node.getparent() @@ -556,7 +552,7 @@ def _parseAndAddDefine(node): except pypa.ParseException: return - iden = ''.join(map(str, res[0])) + iden = ''.join(list(map(str, res[0]))) expn = res[-1] para = res[1:-1] __macrofuncs[iden] = (para, expn) @@ -580,58 +576,57 @@ def _getFeatures(root, options): """ def _wrapGrOuterUp(fouter, featuresgrouter, eelem): - itouter = fouter[-1] # feature surround tags + itouter = fouter[-1] # feature surround tags fouter = fouter[:-1] selem = itouter[0][1] for (sig, _) in itouter: featuresgrouter.append((sig, selem, eelem)) - return (fouter, featuresgrouter) - + return fouter, featuresgrouter def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # wrap up the feature - if (not flist): + if not flist: raise IfdefEndifMismatchError() - itsig = flist[-1] # feature signature + itsig = flist[-1] # feature signature flist = flist[:-1] - itcode = fcode[-1] # feature code + itcode = fcode[-1] # feature code itcode = itcode.replace('\n\n', '\n') - itcode = itcode[1:] # itcode starts with '\n'; del + itcode = itcode[1:] # itcode starts with '\n'; del fcode = fcode[:-1] - itinner = finner[-1] # feature enclosed tags + itinner = finner[-1] # feature enclosed tags finner = finner[:-1] # handle the feature code - if (features.has_key(itsig)): + if itsig in features: features[itsig][1].append(itcode) else: - features[itsig] = (len(flist)+1, [itcode]) + features[itsig] = (len(flist) + 1, [itcode]) # handle the inner granularity featuresgrinner.append((itsig, itinner)) - return (features, featuresgrinner, fcode, flist, finner) - - features = {} # see above; return value - featuresgrinner = [] # see above; return value - featuresgrouter = [] # see above; return value - flist = [] # holds the features in order - # list empty -> no features to parse - # list used as a stack - # last element = top of stack; - # and the element we currently - # collecting source-code lines for - fouter = [] # holds the xml-nodes of the ifdefs/endifs - # in order like flist - fcode = [] # holds the code of the features in - # order like flist - finner = [] # holds the tags of the features in - # order like flist - condinhist = [] # order of the conditional includes - # with feature names - parcon = False # parse-conditional-flag - parend = False # parse-endif-flag - _ = 0 # else and elif depth + return features, featuresgrinner, fcode, flist, finner + + features = {} # see above; return value + featuresgrinner = [] # see above; return value + featuresgrouter = [] # see above; return value + flist = [] # holds the features in order + # list empty -> no features to parse + # list used as a stack + # last element = top of stack; + # and the element we currently + # collecting source-code lines for + fouter = [] # holds the xml-nodes of the ifdefs/endifs + # in order like flist + fcode = [] # holds the code of the features in + # order like flist + finner = [] # holds the tags of the features in + # order like flist + condinhist = [] # order of the conditional includes + # with feature names + parcon = False # parse-conditional-flag + parend = False # parse-endif-flag + _ = 0 # else and elif depth # iterate over all tags separately - and -tag for event, elem in etree.iterwalk(root, events=("start", "end")): @@ -641,13 +636,13 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # hitting on conditional-macro if ((tag in __conditionals_all) and (event == 'start') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = True # hitting next conditional macro; any of ifdef, else or elif if ((tag in __conditionals_all) and (event == 'end') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = False # with else or elif we finish up the last if, therefor @@ -655,15 +650,18 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): if ((tag in __conditionals_else) or (tag in __conditionals_elif)): (features, featuresgrinner, - fcode, flist, finner) = _wrapFeatureUp(features, - featuresgrinner, fcode, flist, finner) + fcode, flist, finner) = _wrapFeatureUp(features, + featuresgrinner, fcode, flist, finner) fname = _getMacroSignature(elem) - if fname: condinhist.append((tag, fname)) - else: condinhist.append((tag, '')) + if fname: + condinhist.append((tag, fname)) + else: + condinhist.append((tag, '')) fsig = _getFeatureSignature(condinhist, options) - if (tag in __conditionals): fouter.append([]) + if tag in __conditionals: + fouter.append([]) fouter[-1] += ([(fsig, elem)]) flist.append(fsig) fcode.append('') @@ -676,9 +674,7 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): parcon = False # hitting end-tag of define-macro - if ((tag in __macro_define) \ - and (event == 'end') \ - and (ns == __cppnscpp)): + if (tag in __macro_define) and (event == 'end') and (ns == __cppnscpp): _parseAndAddDefine(elem) # iterateting in subtree of conditional-node @@ -687,24 +683,18 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # handling endif-macro # hitting an endif-macro start-tag - if ((tag in __conditionals_endif) \ - and (event == "start") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "start") and (ns == __cppnscpp): # check the cpp:namespace parend = True # hitting the endif-macro end-tag - if ((tag in __conditionals_endif) \ - and (event == "end") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "end") and (ns == __cppnscpp): # check the cpp:namespace parend = False (features, featuresgrinner, fcode, flist, finner) = \ - _wrapFeatureUp(features, featuresgrinner, - fcode, flist, finner) - (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, - featuresgrouter, elem) + _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner) + (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, featuresgrouter, elem) - while (condinhist[-1][0] != 'if'): + while condinhist[-1][0] != 'if': condinhist = condinhist[:-1] condinhist = condinhist[:-1] @@ -713,18 +703,18 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): continue # collect the source-code of the feature - if (len(flist)): - if ((event == "start") and (elem.text)): + if len(flist): + if (event == "start") and elem.text: fcode[-1] += elem.text - if ((event == "end") and (elem.tail)): + if (event == "end") and elem.tail: fcode[-1] += elem.tail - if (ns == __cppnsdef or tag not in __conditionals_all): + if ns == __cppnsdef or tag not in __conditionals_all: finner[-1].append((tag, event, elem.sourceline)) - if (flist): + if flist: raise IfdefEndifMismatchError() - return (features, featuresgrinner, featuresgrouter) + return features, featuresgrinner, featuresgrouter def _getOuterGranularity(fnodes): @@ -735,7 +725,7 @@ def _getOuterGranularity(fnodes): grouter = list() for (sig, selem, _) in fnodes: - tags = _getASTHistory(selem)[:-1] # cut of unit-tag + tags = _getASTHistory(selem)[:-1] # cut of unit-tag grouter.append((sig, tags, selem.sourceline)) return grouter @@ -751,93 +741,102 @@ def _getOuterGranularityStats(lgran): """ gotopbgr = 0 gofunbgr = 0 - gostrbrl = 0 # local - gostrbrg = 0 # global + gostrbrl = 0 # local + gostrbrg = 0 # global goinnbgr = 0 goexpbgr = 0 gostmbgr = 0 gopambgr = 0 - goerror = 0 + goerror = 0 for (_, gran, line) in lgran: if len(gran) == 0: gotopbgr += 1 continue if gran[0] in ['block']: - if len(gran) == 1: # configure the method signature + if len(gran) == 1: # configure the method signature gofunbgr += 1 continue if gran[1] in ['function', 'extern', 'block']: gofunbgr += 1 elif gran[1] in ['struct', 'union', - 'enum']: # test_struct_union_enum.c - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + 'enum']: # test_struct_union_enum.c + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 elif gran[1] in ['expr']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 elif gran[1] in ['while', 'for', 'then', 'do', - 'else', 'switch', 'case', 'default']: - goinnbgr += 1 # test_loop.c - elif gran[1] in ['decl']: # test_struct_union_enum.c - if 'function' in gran[3:]: gostrbrl += 1 - else: gostrbrg += 1 + 'else', 'switch', 'case', 'default']: + goinnbgr += 1 # test_loop.c + elif gran[1] in ['decl']: # test_struct_union_enum.c + if 'function' in gran[3:]: + gostrbrl += 1 + else: + gostrbrg += 1 else: print('ERROR: gran (%s) at this ' - 'level unknown (line %s)' % (gran, line)) + 'level unknown (line %s)' % (gran, line)) goerror += 1 continue elif gran[0] in ['expr']: - if gran[1] in ['expr_stmt']: # test_stmt.c + if gran[1] in ['expr_stmt']: # test_stmt.c gostmbgr += 1 - elif gran[1] in ['condition', 'return']: # test_condition.c + elif gran[1] in ['condition', 'return']: # test_condition.c goexpbgr += 1 - elif gran[1] in ['argument']: # test_call.c + elif gran[1] in ['argument']: # test_call.c gostmbgr += 1 elif gran[1] in ['block']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 - elif gran[1] in ['init', 'index']: # test_stmt.c + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 + elif gran[1] in ['init', 'index']: # test_stmt.c gostmbgr += 1 else: print('ERROR: gran (%s) at this level' - 'unknown (line %s)' % (gran, line)) + 'unknown (line %s)' % (gran, line)) goerror += 1 continue - elif gran[0] in ['while', 'do']: # test_loop.c + elif gran[0] in ['while', 'do']: # test_loop.c goinnbgr += 1 continue elif gran[0] in ['expr_stmt'] and len(gran) == 1: gostmbgr += 1 continue elif gran[:3] == ['expr_stmt', 'block', 'struct']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 continue - elif gran[0] in ['decl_stmt']: # test_stmt.c + elif gran[0] in ['decl_stmt']: # test_stmt.c gostmbgr += 1 continue - elif gran[0] in ['condition']: # test_condition.c + elif gran[0] in ['condition']: # test_condition.c goexpbgr += 1 continue elif gran[0] in ['if', 'else', 'case', 'default', - 'then', 'for']: # test_condition.c + 'then', 'for']: # test_condition.c goinnbgr += 1 continue elif gran[0] in ['parameter_list', - 'argument_list']: # test_call.c + 'argument_list']: # test_call.c gopambgr += 1 continue elif gran[0] in ['argument'] and gran[1] in ['argument_list']: gostmbgr += 1 - elif gran[0] in ['init'] and gran[1] in ['decl']: # test_stmt.c + elif gran[0] in ['init'] and gran[1] in ['decl']: # test_stmt.c gostmbgr += 1 continue - elif gran[0] in ['function']: # function prototype + elif gran[0] in ['function']: # function prototype continue else: - print('ERROR: outer granularity (%s, %s) not recognized!' % \ - (gran, line)) + print(f'ERROR: outer granularity ({gran}, {line}) not recognized!') goerror += 1 return (gotopbgr, gofunbgr, gostrbrl, gostrbrg, @@ -867,22 +866,23 @@ def _getInnerGranularityStats(igran): for (_, gran) in igran: for (tag, event, line) in gran: - if (skiptilltag != ''): + if skiptilltag != '': if (tag == skiptilltag[0] and event == 'end' and line == skiptilltag[2]): skiptilltag = '' continue - if tag in ['name', 'endif']: continue + if tag in ['name', 'endif']: + continue elif tag in ['define', 'directive', 'include', - 'macro', 'undef']: + 'macro', 'undef']: gmacrogr += 1 elif tag in ['struct', 'union', 'enum', 'function', 'extern', - 'function_decl', 'decl_stmt', 'typedef']: + 'function_decl', 'decl_stmt', 'typedef']: ginablgr += 1 elif tag in ['if', 'while', 'return', 'then', - 'for', 'do', 'case', 'else', 'block']: + 'for', 'do', 'case', 'else', 'block']: giunblgr += 1 elif tag in ['param', 'argument']: gipambgr += 1 @@ -894,12 +894,13 @@ def _getInnerGranularityStats(igran): gistmbgr += 1 else: print('ERROR: inner granularity (%s, %s, %s)) ' - 'not recognized!' % (tag, event, line)) + 'not recognized!' % (tag, event, line)) gierror += 1 continue - if event == 'start': skiptilltag = (tag, event, line) + if event == 'start': + skiptilltag = (tag, event, line) - return (gmacrogr, ginablgr, giunblgr, giexpbgr, gistmbgr, gierror) + return gmacrogr, ginablgr, giunblgr, giexpbgr, gistmbgr, gierror def _getFeatureStats(features): @@ -915,30 +916,30 @@ def _getFeatureStats(features): """ lof = 0 nod = 0 - lofmin = -1 # feature-code can be empty + lofmin = -1 # feature-code can be empty lofmax = 0 lofmean = 0 lofstd = 0 nof = len(features.keys()) tmp = [item for (_, item) in features.itervalues()] tmp = _flatten(tmp) - floflist = map(lambda n: n.count('\n'), tmp) + floflist = list(map(lambda n: n.count('\n'), tmp)) - if (len(floflist)): + if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = reduce(lambda m,n: m+n, floflist) - lofmean = pstat.stats.lmean(floflist) + lof = reduce(lambda m, n: m + n, floflist) + lofmean = np.mean(floflist) - if (len(floflist) > 1): - lofstd = pstat.stats.lstdev(floflist) + if len(floflist) > 1: + lofstd = np.std(floflist) - return (nof, nod, lof, lofmin, lofmax, lofmean, lofstd) + return nof, nod, lof, lofmin, lofmax, lofmean, lofstd def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = filter(lambda (sig, (depth, code)): depth == 1, features.iteritems()) + nof1 = list(filter(lambda t: t[1][0] == 1, features.iteritems())) # t = (sig, (depth, code)) return nof1 @@ -966,13 +967,13 @@ def _compareFeatureCode(fcode): """ fcodes = set(fcode) - if (len(fcodes) == 1 and len(fcode) > 1): + if len(fcodes) == 1 and len(fcode) > 1: return "hom" - if (len(fcode) == len(fcodes)): + if len(fcode) == len(fcodes): return "het" - if (len(fcode) > len(fcodes)): + if len(fcode) > len(fcodes): return "hethom" scode = {} @@ -985,27 +986,27 @@ def _compareFeatureCode(fcode): for (key, (_, item)) in features.iteritems(): # distinguish according to feature-signature # shared code - if ('||' in key and (not '&&' in key)): + if '||' in key and ('&&' not in key): scode[key] = item # derivative only && - if ('&&' in key and (not '||' in key)): + if '&&' in key and ('||' not in key): deriv[key] = item # combination shared code and derivative - if ('&&' in key and '||' in key): + if '&&' in key and '||' in key: desc[key] = item # distinguish according to feature-code ret = _compareFeatureCode(item) - if (ret == "het"): + if ret == "het": het[key] = item - if (ret == "hom"): + if ret == "hom": hom[key] = item - if (ret == "hethom"): + if ret == "hethom": hethom[key] = item - return (scode, deriv, desc, het, hom, hethom) + return scode, deriv, desc, het, hom, hethom def _getNumOfDefines(defset): @@ -1021,7 +1022,7 @@ def _getNumOfDefines(defset): # basic operation of this function is to check __defset against # __macrofuncs funcmacros = __macrofuncs.keys() - funcmacros = map(lambda n: n.split('(')[0], funcmacros) + funcmacros = list(map(lambda n: n.split('(')[0], funcmacros)) funcmacros = set(funcmacros) return len((defset - funcmacros)) @@ -1041,27 +1042,35 @@ def _getScatteringTanglingDegrees(sigs, defines): def __add(x, y): """This method is a helper function to add values of lists pairwise. See below for more information.""" - return x+y + return x + y - scat = list() # relation define to signatures - tang = [0]*len(sigs) # signatures overall + scat = list() # relation define to signatures + tang = [0] * len(sigs) # signatures overall for d in defines: - dre = re.compile(r'\b'+d+r'\b') # using word boundaries - vec = map(lambda s: not dre.search(s) is None, sigs) + dre = re.compile(r'\b' + d + r'\b') # using word boundaries + vec = list(map(lambda s: not dre.search(s) is None, sigs)) scat.append(vec.count(True)) - tang = map(__add, tang, vec) + tang = list(map(__add, tang, vec)) - if (len(scat)): sdegmean = pstat.stats.lmean(scat) - else: sdegmean = 0 - if (len(scat) > 1): sdegstd = pstat.stats.lstdev(scat) - else: sdegstd = 0 + if len(scat): + sdegmean = np.mean(scat) + else: + sdegmean = 0 + if len(scat) > 1: + sdegstd = np.std(scat) + else: + sdegstd = 0 - if (len(tang)): tdegmean = pstat.stats.lmean(tang) - else: tdegmean = 0 - if (len(tang) > 1): tdegstd = pstat.stats.lstdev(tang) - else: tdegstd = 0 + if len(tang): + tdegmean = np.mean(tang) + else: + tdegmean = 0 + if len(tang) > 1: + tdegstd = np.std(tang) + else: + tdegstd = 0 - return (sdegmean, sdegstd, tdegmean, tdegstd) + return sdegmean, sdegstd, tdegmean, tdegstd def _getGranularityStats(fcodetags): @@ -1073,20 +1082,20 @@ def _getGranularityStats(fcodetags): expr -> expression """ _interestingtags = [ - 'define', # define statement - 'include', # include statement - 'decl_stmt', # declaration statement - 'expr_stmt', # expression statement - 'function_decl', # function declaration - 'parameter_list', # parameter list - 'param', # parameter + 'define', # define statement + 'include', # include statement + 'decl_stmt', # declaration statement + 'expr_stmt', # expression statement + 'function_decl', # function declaration + 'parameter_list', # parameter list + 'param', # parameter ] - _curskiptag = None # holds the element we are going to skip for - _skipedtags = [] # if we find one of the elements above, we - # start skipping the following tags, until - # the corresponding endtag is found; if we - # do not find the endtag, we are going to - # start analyzing the elements in _skipedtags + _curskiptag = None # holds the element we are going to skip for + _skipedtags = [] # if we find one of the elements above, we + # start skipping the following tags, until + # the corresponding endtag is found; if we + # do not find the endtag, we are going to + # start analyzing the elements in _skipedtags granstats = dict() for tag in _interestingtags: @@ -1096,38 +1105,38 @@ def _getGranularityStats(fcodetags): _curskiptag = None _skipedtags = [] for (tag, _, sourceline) in ftagsl: - if _curskiptag == None and tag in _interestingtags: + if _curskiptag is None and tag in _interestingtags: _curskiptag = tag continue - if _curskiptag != None and tag == _curskiptag: + if _curskiptag is not None and tag == _curskiptag: granstats[tag] = granstats[tag] + 1 _curskiptag = None _skipedtags = [] continue - if _curskiptag != None: + if _curskiptag is not None: _skipedtags.append((tag, sourceline)) continue -# if _skipedtags != []: -# print("ERROR: ", _skipedtags) + # if _skipedtags != []: + # print("ERROR: ", _skipedtags) return granstats def __getNumOfFilesPerFeatureStats(filetofeatureconstants): featureconstantstofiles = dictinvert(filetofeatureconstants) - numbers = map(lambda v: len(v), featureconstantstofiles.values()) + numbers = list(map(lambda v: len(v), featureconstantstofiles.values())) - #mean - if (len(numbers) > 0): - numbersmean = pstat.stats.lmean(numbers) + # mean + if len(numbers) > 0: + numbersmean = np.mean(numbers) else: numbersmean = 0 # std - if (len(numbers) > 1): - numbersstd = pstat.stats.lstdev(numbers) + if len(numbers) > 1: + numbersstd = np.std(numbers) else: numbersstd = 0 - return (numbersmean,numbersstd) + return numbersmean, numbersstd def _checkForEquivalentSig(l, sig): @@ -1140,7 +1149,7 @@ def _checkSigEquivalence(sig1, sig2): It uses an xmlrpc-call on troi.fim.uni-passau.de.""" global __errorfexp global __errormatch - if not (sig1 and sig2): # ommit empty signatures + if not (sig1 and sig2): # ommit empty signatures return False # if options.str: @@ -1153,12 +1162,12 @@ def _checkSigEquivalence(sig1, sig2): raise NoEquivalentSigError() -def resetModule() : +def resetModule(): global __macrofuncs, __defset, __defsetf, __nestedIfdefsLevels - __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" - __defset = set() # macro-objects - __defsetf = dict() # macro-objects per file + __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", + # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" + __defset = set() # macro-objects + __defsetf = dict() # macro-objects per file __nestedIfdefsLevels = [] @@ -1169,8 +1178,8 @@ def apply(folder, options): # overall status variables resetModule() - sigmap = {} # {: []} - afeatures = {} # identified features; {: (depth, [code])} + sigmap = {} # {: []} + afeatures = {} # identified features; {: (depth, [code])} def _mergeFeatures(ffeatures): """This function merges the, with the parameter given @@ -1181,9 +1190,9 @@ def _mergeFeatures(ffeatures): try: sigmatch = _checkForEquivalentSig(sigmap.keys(), psig) (tmpdepth, tmpcode) = afeatures[sigmap[sigmatch][0]] -# if (tmpdepth != depth): -# print("INFO: depths of feature fragments do not" + -# " match (%s, %s)!" % (str(tmpdepth), str(depth))) + # if (tmpdepth != depth): + # print("INFO: depths of feature fragments do not" + + # " match (%s, %s)!" % (str(tmpdepth), str(depth))) tmpdepth = min(tmpdepth, depth) tmpcode += code afeatures[sigmap[sigmatch][0]] = (tmpdepth, tmpcode) @@ -1202,7 +1211,7 @@ def _mergeFeatures(ffeatures): fcount = 0 files = returnFileNames(folder, ['.xml']) files.sort() - fstats = [None]*len(__statsorder) + fstats = [None] * len(__statsorder) ftotal = len(files) # get statistics for all files; write results into csv @@ -1213,14 +1222,14 @@ def _mergeFeatures(ffeatures): try: tree = etree.parse(file) except etree.XMLSyntaxError: - print("ERROR: cannot parse (%s). Skipping this file." % os.path.join(folder, file)) + print(f"ERROR: cannot parse ({os.path.join(folder, file)}). Skipping this file.") continue root = tree.getroot() try: (features, _, featuresgrouter) = _getFeatures(root, options) except IfdefEndifMismatchError: - print("ERROR: ifdef-endif mismatch in file (%s)" % (os.path.join(folder, file))) + print(f"ERROR: ifdef-endif mismatch in file ({os.path.join(folder, file)})") continue _mergeFeatures(features) @@ -1232,23 +1241,23 @@ def _mergeFeatures(ffeatures): # granularity stats grouter = _getOuterGranularity(featuresgrouter) (gotopbgr, gofunbgr, gostrbrl, gostrbrg, - goinnbgr, goexpbgr, gostmbgr, gopambgr, goerror) = \ - _getOuterGranularityStats(grouter) + goinnbgr, goexpbgr, gostmbgr, gopambgr, goerror) = \ + _getOuterGranularityStats(grouter) fstats[__statsorder.GRANGL.value] = gotopbgr - fstats[__statsorder.GRANFL.value] = gofunbgr+gostrbrl+gostrbrg + fstats[__statsorder.GRANFL.value] = gofunbgr + gostrbrl + gostrbrg fstats[__statsorder.GRANBL.value] = goinnbgr fstats[__statsorder.GRANEL.value] = goexpbgr fstats[__statsorder.GRANSL.value] = gostmbgr fstats[__statsorder.GRANML.value] = gopambgr fstats[__statsorder.GRANERR.value] = goerror - #adjust file name if wanted - if options.filenamesRelative : # relative file name (root is project folder (not included in path)) + # adjust file name if wanted + if options.filenamesRelative: # relative file name (root is project folder (not included in path)) file = os.path.relpath(file, folder) - if options.filenames == options.FILENAME_SRCML : # cppstats file names - pass # nothing to do here, as the file path is the cppstats path by default - if options.filenames == options.FILENAME_SOURCE : # source file name + if options.filenames == options.FILENAME_SRCML: # cppstats file names + pass # nothing to do here, as the file path is the cppstats path by default + if options.filenames == options.FILENAME_SOURCE: # source file name file = file.replace(".xml", "").replace("/_cppstats/", "/source/", 1) # general stats @@ -1259,17 +1268,17 @@ def _mergeFeatures(ffeatures): fstats[__statsorder.NDMAX.value] = ndmax tmp = [it for it in root.iterdescendants()] - if (len(tmp)): floc = tmp[-1].sourceline - else: floc = 0 + if len(tmp): + floc = tmp[-1].sourceline + else: + floc = 0 fstats[__statsorder.LOC.value] = floc # feature-amount - (_, _, lof, _, _, _, _) = \ - _getFeatureStats(features) - if __defsetf.has_key(__curfile): - fstats[__statsorder.NOFC.value] = \ - _getNumOfDefines(__defsetf[__curfile]) + (_, _, lof, _, _, _, _) = _getFeatureStats(features) + if __curfile in __defsetf: + fstats[__statsorder.NOFC.value] = _getNumOfDefines(__defsetf[__curfile]) else: fstats[__statsorder.NOFC.value] = 0 fstats[__statsorder.LOF.value] = lof @@ -1280,84 +1289,70 @@ def _mergeFeatures(ffeatures): fdcsv.writerow(fstats) - # writing convinience functions - fnum = fcount + 1 # +1 for the header of the table + fnum = fcount + 1 # +1 for the header of the table excelcols = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', - 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', - 'V', 'W', 'X', 'Y', 'Z', 'AA', 'AB', 'AC', 'AD', 'AE', - 'AF', 'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', - 'AO', 'AP', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AV', 'AW', - 'AX', 'AY', 'AZ', 'BA', 'BB', 'BC', 'BD', 'BE', 'BF', - 'BG', 'BH', 'BI', 'BJ', 'BK', 'BL', 'BM', 'BN', 'BO', - 'BP', 'BQ', 'BR', 'BS', 'BT', 'BU', 'BV', 'BW', 'BX', - 'BY', 'BZ'] + 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', + 'V', 'W', 'X', 'Y', 'Z', 'AA', 'AB', 'AC', 'AD', 'AE', + 'AF', 'AG', 'AH', 'AI', 'AJ', 'AK', 'AL', 'AM', 'AN', + 'AO', 'AP', 'AQ', 'AR', 'AS', 'AT', 'AU', 'AV', 'AW', + 'AX', 'AY', 'AZ', 'BA', 'BB', 'BC', 'BD', 'BE', 'BF', + 'BG', 'BH', 'BI', 'BJ', 'BK', 'BL', 'BM', 'BN', 'BO', + 'BP', 'BQ', 'BR', 'BS', 'BT', 'BU', 'BV', 'BW', 'BX', + 'BY', 'BZ'] # FIXME with separator line, functions must start in line 3! (other scripts, too?) - excelfunc = [None]*len(__statsorder) + excelfunc = [None] * len(__statsorder) excelfunc[__statsorder.FILENAME.value] = "FUNCTIONS" - excelfunc[__statsorder.LOC.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.LOC.value], \ - excelcols[__statsorder.LOC.value], fnum) - excelfunc[__statsorder.LOF.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.LOF.value], \ - excelcols[__statsorder.LOF.value], fnum) - excelfunc[__statsorder.ANDAVG.value] = "=SUM(%s2:%s%s)/countif(%s2:%s%s;\">0\")" % \ - (excelcols[__statsorder.ANDAVG.value], \ - excelcols[__statsorder.ANDAVG.value], fnum, \ - excelcols[__statsorder.ANDAVG.value], \ - excelcols[__statsorder.ANDAVG.value], fnum) - excelfunc[__statsorder.ANDSTDEV.value] = "=SUM(%s2:%s%s)/countif(%s2:%s%s;\">0\")" % \ - (excelcols[__statsorder.ANDSTDEV.value], \ - excelcols[__statsorder.ANDSTDEV.value], fnum, \ - excelcols[__statsorder.ANDAVG.value], # it might be that mean 1 std 0 - excelcols[__statsorder.ANDAVG.value], fnum) # therefore we use mean here - excelfunc[__statsorder.GRANGL.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.GRANGL.value], \ - excelcols[__statsorder.GRANGL.value], fnum) - excelfunc[__statsorder.GRANFL.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.GRANFL.value], \ - excelcols[__statsorder.GRANFL.value], fnum) - excelfunc[__statsorder.GRANBL.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.GRANBL.value], \ - excelcols[__statsorder.GRANBL.value], fnum) - excelfunc[__statsorder.GRANEL.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.GRANEL.value], \ - excelcols[__statsorder.GRANEL.value], fnum) - excelfunc[__statsorder.GRANSL.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.GRANSL.value], \ - excelcols[__statsorder.GRANSL.value], fnum) - excelfunc[__statsorder.GRANML.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.GRANML.value], \ - excelcols[__statsorder.GRANML.value], fnum) - excelfunc[__statsorder.GRANERR.value] = "=SUM(%s2:%s%s)" % \ - (excelcols[__statsorder.GRANERR.value], \ - excelcols[__statsorder.GRANERR.value], fnum) - excelfunc[__statsorder.NDMAX.value] = "=MAX(%s2:%s%s)" % \ - (excelcols[__statsorder.NDMAX.value], \ - excelcols[__statsorder.NDMAX.value], fnum) + excelfunc[__statsorder.LOC.value] = \ + f"=SUM({excelcols[__statsorder.LOC.value]}2:{excelcols[__statsorder.LOC.value]}{fnum})" + excelfunc[__statsorder.LOF.value] = \ + f"=SUM({excelcols[__statsorder.LOF.value]}2:{excelcols[__statsorder.LOF.value]}{fnum})" + excelfunc[__statsorder.ANDAVG.value] = \ + f"=SUM({excelcols[__statsorder.ANDAVG.value]}2:{excelcols[__statsorder.ANDAVG.value]}{fnum})" \ + f"/countif({excelcols[__statsorder.ANDAVG.value]}2:{excelcols[__statsorder.ANDAVG.value]}{fnum};\">0\")" + excelfunc[__statsorder.ANDSTDEV.value] = \ + f"=SUM({excelcols[__statsorder.ANDSTDEV.value]}2:{excelcols[__statsorder.ANDSTDEV.value]}{fnum})" \ + f"/countif({excelcols[__statsorder.ANDAVG.value]}2:" \ + f"{excelcols[__statsorder.ANDAVG.value]}{fnum};\">0\")" # therefore we use mean here + excelfunc[__statsorder.GRANGL.value] = \ + f"=SUM({excelcols[__statsorder.GRANGL.value]}2:{excelcols[__statsorder.GRANGL.value]}{fnum})" + excelfunc[__statsorder.GRANFL.value] = \ + f"=SUM({excelcols[__statsorder.GRANFL.value]}2:{excelcols[__statsorder.GRANFL.value]}{fnum})" + excelfunc[__statsorder.GRANBL.value] = \ + f"=SUM({excelcols[__statsorder.GRANBL.value]}2:{excelcols[__statsorder.GRANBL.value]}{fnum})" + excelfunc[__statsorder.GRANEL.value] = \ + f"=SUM({excelcols[__statsorder.GRANEL.value]}2:{excelcols[__statsorder.GRANEL.value]}{fnum})" + excelfunc[__statsorder.GRANSL.value] = \ + f"=SUM({excelcols[__statsorder.GRANSL.value]}2:{excelcols[__statsorder.GRANSL.value]}{fnum})" + excelfunc[__statsorder.GRANML.value] = \ + f"=SUM({excelcols[__statsorder.GRANML.value]}2:{excelcols[__statsorder.GRANML.value]}{fnum})" + excelfunc[__statsorder.GRANERR.value] = \ + f"=SUM({excelcols[__statsorder.GRANERR.value]}2:{excelcols[__statsorder.GRANERR.value]}{fnum})" + excelfunc[__statsorder.NDMAX.value] = \ + f"=MAX({excelcols[__statsorder.NDMAX.value]}2:{excelcols[__statsorder.NDMAX.value]}{fnum})" fdcsv.writerow(excelfunc) # overall - stats - astats = [None]*len(__statsorder) + astats = [None] * len(__statsorder) # LOF (_, _, lof, _, _, _, _) = \ - _getFeatureStats(afeatures) + _getFeatureStats(afeatures) # SDEG + TDEG sigs = _flatten(sigmap.values()) defs = list(__defset) (sdegmean, sdegstd, tdegmean, tdegstd) = \ - _getScatteringTanglingDegrees(sigs,defs) + _getScatteringTanglingDegrees(sigs, defs) # ANDAVG + ANDSTDEV nestedIfdefsLevels = _flatten(__nestedIfdefsLevels) - if (len(nestedIfdefsLevels)): - nnimean = pstat.stats.lmean(nestedIfdefsLevels) + if len(nestedIfdefsLevels): + nnimean = np.mean(nestedIfdefsLevels) else: nnimean = 0 - if (len(nestedIfdefsLevels) > 1): - nnistd = pstat.stats.lstdev(nestedIfdefsLevels) + if len(nestedIfdefsLevels) > 1: + nnistd = np.std(nestedIfdefsLevels) else: nnistd = 0 @@ -1391,12 +1386,12 @@ def _mergeFeatures(ffeatures): # add command line options def addCommandLineOptionsMain(optionparser): - ''' add command line options for a direct call of this script''' + """ add command line options for a direct call of this script""" optionparser.add_argument("--folder", dest="folder", - help="input folder [default=%(default)s]", default=".") + help="input folder [default=%(default)s]", default=".") -def addCommandLineOptions(optionparser) : +def addCommandLineOptions(optionparser): # TODO implement CSP solving? # optionparser.add_option("--csp", dest="csp", action="store_true", # default=False, help="make use of csp solver to check " \ @@ -1417,7 +1412,6 @@ def getResultsFile(): ################################################## if __name__ == '__main__': - ################################################## # options parsing parser = ArgumentParser(formatter_class=RawTextHelpFormatter) @@ -1430,7 +1424,7 @@ def getResultsFile(): # main folder = os.path.abspath(options.folder) - if (os.path.isdir(folder)): + if os.path.isdir(folder): apply(folder, options) else: sys.exit(-1) From da347fe6df01054d9f4d1dd4e1b3351a28259a4f Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 11:27:42 +0100 Subject: [PATCH 05/72] Fix analyses/generalvalues.py --- analyses/generalvalues.py | 352 +++++++++++++++++++------------------- 1 file changed, 173 insertions(+), 179 deletions(-) diff --git a/analyses/generalvalues.py b/analyses/generalvalues.py index dda1a4a..7e37aa3 100644 --- a/analyses/generalvalues.py +++ b/analyses/generalvalues.py @@ -27,30 +27,27 @@ import os import re import sys -import xmlrpclib from argparse import ArgumentParser, RawTextHelpFormatter from collections import OrderedDict - # ################################################# # path adjustments, so that all imports can be done relative to these paths __lib_subfolder = "lib" sys.path.append(os.path.abspath(__lib_subfolder)) # lib subfolder - # ################################################# # external modules # python-lxml module from lxml import etree # statistics module -from statlib import pstat +import numpy as np # pyparsing module import pyparsing as pypa -pypa.ParserElement.enablePackrat() # speed up parsing -sys.setrecursionlimit(8000) # handle larger expressions +pypa.ParserElement.enablePackrat() # speed up parsing +sys.setrecursionlimit(8000) # handle larger expressions ################################################## # config: @@ -80,21 +77,21 @@ __conditionals_ending = __conditionals_elif + __conditionals_else + \ __conditionals_endif __macro_define = ['define'] -__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" -__curfile = '' # current processed xml-file -__defset = set() # macro-objects -__defsetf = dict() # macro-objects per file +__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", +# used as "GLIBVERSION(x,y,z) 100*x+10*y+z" +__curfile = '' # current processed xml-file +__defset = set() # macro-objects +__defsetf = dict() # macro-objects per file -################################################## +################################################## ################################################## # helper functions, constants and errors -def returnFileNames(folder, extfilt = ['.xml']): - '''This function returns all files of the input folder - and its subfolders.''' +def returnFileNames(folder, extfilt=['.xml']): + """This function returns all files of the input folder + and its subfolders.""" filesfound = list() if os.path.isdir(folder): @@ -105,29 +102,29 @@ def returnFileNames(folder, extfilt = ['.xml']): wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) + tmpfiles) tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles) filesfound += tmpfiles tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders) wqueue += tmpfolders return filesfound -def _prologCSV(folder, file, headings, delimiter = ","): +def _prologCSV(folder, file, headings, delimiter=","): """prolog of the CSV-output file no corresponding _epilogCSV.""" fd = open(os.path.join(folder, file), 'w') fdcsv = csv.writer(fd, delimiter=delimiter) fdcsv.writerow(["sep=" + delimiter]) fdcsv.writerow(headings) - return (fd, fdcsv) + return fd, fdcsv def _flatten(l): @@ -142,7 +139,7 @@ def _flatten(l): i -= 1 break else: - l[i:i+1] = l[i] + l[i:i + 1] = l[i] i += 1 return l @@ -154,10 +151,10 @@ def _collectDefines(d): but not #define GLIBCVER(x,y,z) ... """ __defset.add(d[0]) - if __defsetf.has_key(__curfile): + if __curfile in __defsetf: __defsetf[__curfile].add(d[0]) else: - __defsetf[__curfile] = set([d[0]]) + __defsetf[__curfile] = {d[0]} return d @@ -173,41 +170,42 @@ def _collectDefines(d): __string = pypa.QuotedString('\'', '\\') __hexadec = \ - pypa.Literal('0x').suppress() + \ - pypa.Word(pypa.hexnums).\ - setParseAction(lambda t: str(int(t[0], 16))) + \ - pypa.Optional(__numlitu) + \ - pypa.Optional(__numlitl) + \ - pypa.Optional(__numlitl) + pypa.Literal('0x').suppress() + \ + pypa.Word(pypa.hexnums).setParseAction(lambda t: str(int(t[0], 16))) + \ + pypa.Optional(__numlitu) + \ + pypa.Optional(__numlitl) + \ + pypa.Optional(__numlitl) __integer = \ - pypa.Optional('~') + \ - pypa.Word(pypa.nums+'-').setParseAction(lambda t: str(int(t[0]))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('U'))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + pypa.Optional('~') + \ + pypa.Word(pypa.nums + '-').setParseAction(lambda t: str(int(t[0]))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('U'))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('L'))) __identifier = \ - pypa.Word(pypa.alphanums+'_'+'-'+'@'+'$').setParseAction(_collectDefines) -__arg = pypa.Word(pypa.alphanums+'_') -__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + \ - __arg) + pypa.Word(pypa.alphanums + '_' + '-' + '@' + '$').setParseAction(_collectDefines) +__arg = pypa.Word(pypa.alphanums + '_') +__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + __arg) __fname = pypa.Word(pypa.alphas, pypa.alphanums + '_') -__function = pypa.Group(__fname + pypa.Literal('(').suppress() + \ - __args + pypa.Literal(')').suppress()) +__function = pypa.Group(__fname + pypa.Literal('(').suppress() + __args + pypa.Literal(')').suppress()) class NoEquivalentSigError(Exception): def __init__(self): pass + def __str__(self): - return ("No equivalent signature found!") + return "No equivalent signature found!" + class IfdefEndifMismatchError(Exception): def __init__(self): pass + def __str__(self): - return ("Ifdef and endif do not match!") + return "Ifdef and endif do not match!" + ################################################## @@ -234,26 +232,26 @@ def _parseFeatureSignatureAndRewrite(sig): # if no equivalence can be found a name rewriting is done # e.g. 'defined' __pt = { - #'defined' : 'defined_', - 'defined' : '', - '!' : '¬', + # 'defined' : 'defined_', + 'defined': '', + '!': '¬', '&&': '&and', '||': '&or', - '<' : '<', - '>' : '>', + '<': '<', + '>': '>', '<=': '<=', '>=': '>=', '==': '=', '!=': '!=', - '*' : '*', # needs rewriting with parenthesis - '/' : '/', - '%' : '', # needs rewriting a % b => modp(a, b) - '+' : '+', - '-' : '-', - '&' : '', # needs rewriting a & b => BitAnd(a, b) - '|' : '', # needs rewriting a | b => BitOr(a, b) - '>>': '>>', # needs rewriting a >> b => a / (2^b) - '<<': '<<', # needs rewriting a << b => a * (2^b) + '*': '*', # needs rewriting with parenthesis + '/': '/', + '%': '', # needs rewriting a % b => modp(a, b) + '+': '+', + '-': '-', + '&': '', # needs rewriting a & b => BitAnd(a, b) + '|': '', # needs rewriting a | b => BitOr(a, b) + '>>': '>>', # needs rewriting a >> b => a / (2^b) + '<<': '<<', # needs rewriting a << b => a * (2^b) } def _rewriteOne(param): @@ -263,8 +261,7 @@ def _rewriteOne(param): ret = __pt[param[0][0]] + '(' + str(param[0][1]) + ')' if param[0][0] == 'defined': ret = __pt[param[0][0]] + str(param[0][1]) - return ret - + return ret def _rewriteTwo(param): """This function returns each two parameter function @@ -280,13 +277,12 @@ def _rewriteTwo(param): ret = '(true &and ' + ret + ')' return ret - operand = __string | __hexadec | __integer | \ - __function | __identifier + operand = __string | __hexadec | __integer | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') expr = pypa.operatorPrecedence(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), - ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), + ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), (compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), ('&&', 2, pypa.opAssoc.LEFT, _rewriteTwo), @@ -295,16 +291,14 @@ def _rewriteTwo(param): try: rsig = expr.parseString(sig)[0] - except pypa.ParseException, e: - print('ERROR (parse): cannot parse sig (%s) -- (%s)' % - (sig, e.col)) + except pypa.ParseException as e: + print(f'ERROR (parse): cannot parse sig ({sig}) -- ({e.col})') return sig except RuntimeError: - print('ERROR (time): cannot parse sig (%s)' % (sig)) + print(f'ERROR (time): cannot parse sig ({sig})') return sig - except ValueError, e: - print('ERROR (parse): cannot parse sig (%s) ~~ (%s)' % - (sig, e)) + except ValueError as e: + print(f'ERROR (parse): cannot parse sig ({sig}) ~~ ({e})') return sig return ''.join(rsig) @@ -325,9 +319,9 @@ def _getMacroSignature(ifdefnode): # get either the expr or the name tag, # which is always the second descendant - if (tag in ['if', 'elif', 'ifdef', 'ifndef']): + if tag in ['if', 'elif', 'ifdef', 'ifndef']: nexpr = [itex for itex in ifdefnode.iterdescendants()] - if (len(nexpr) == 1): + if len(nexpr) == 1: res = nexpr[0].tail else: nexpr = nexpr[1] @@ -336,6 +330,8 @@ def _getMacroSignature(ifdefnode): _elsePrefix = "###" + + def _getFeatureSignature(condinhist, options): """This method returns a feature signature that belongs to the current history of conditional inclusions held in condinhist.""" @@ -343,7 +339,7 @@ def _getFeatureSignature(condinhist, options): # signature; reason is elements like else or elif, which mean # basically invert the fname found before # rewritelist = [(tag, fname, )] - rewritelist = [None]*len(condinhist) + rewritelist = [None] * len(condinhist) cur = -1 for tag, fname in condinhist: @@ -351,15 +347,15 @@ def _getFeatureSignature(condinhist, options): if tag == 'if': rewritelist[cur] = (tag, fname, False) if tag in ['elif', 'else']: - (t, f, _) = rewritelist[cur-1] - rewritelist[cur-1] = (t, f, True) + (t, f, _) = rewritelist[cur - 1] + rewritelist[cur - 1] = (t, f, True) rewritelist[cur] = (tag, fname, False) fsig = '' for (tag, fname, invert) in rewritelist: if invert: - if (options.rewriteifdefs): + if options.rewriteifdefs: fname = '!(' + fname + ')' else: fname = _elsePrefix + '!(' + fname + ')' @@ -369,8 +365,8 @@ def _getFeatureSignature(condinhist, options): continue if tag == 'else': continue - if tag in [ 'if', 'elif']: - if (options.rewriteifdefs): + if tag in ['if', 'elif']: + if options.rewriteifdefs: fsig = '(' + fsig + ') && (' + fname + ')' else: fsig = fname @@ -417,59 +413,58 @@ def _getFeatures(root, options): """ def _wrapGrOuterUp(fouter, featuresgrouter, eelem): - itouter = fouter[-1] # feature surround tags + itouter = fouter[-1] # feature surround tags fouter = fouter[:-1] selem = itouter[0][1] for (sig, _) in itouter: featuresgrouter.append((sig, selem, eelem)) - return (fouter, featuresgrouter) - + return fouter, featuresgrouter def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # wrap up the feature - if (not flist): + if not flist: raise IfdefEndifMismatchError() - itsig = flist[-1] # feature signature + itsig = flist[-1] # feature signature flist = flist[:-1] - itcode = fcode[-1] # feature code + itcode = fcode[-1] # feature code itcode = itcode.replace('\n\n', '\n') - itcode = itcode[1:] # itcode starts with '\n'; del + itcode = itcode[1:] # itcode starts with '\n'; del fcode = fcode[:-1] - itinner = finner[-1] # feature enclosed tags + itinner = finner[-1] # feature enclosed tags finner = finner[:-1] # handle the feature code - if (features.has_key(itsig)): + if itsig in features: features[itsig][1].append(itcode) else: - features[itsig] = (len(flist)+1, [itcode]) + features[itsig] = (len(flist) + 1, [itcode]) # handle the inner granularity featuresgrinner.append((itsig, itinner)) - return (features, featuresgrinner, fcode, flist, finner) + return features, featuresgrinner, fcode, flist, finner from collections import OrderedDict - features = OrderedDict({}) # see above; return value - featuresgrinner = [] # see above; return value - featuresgrouter = [] # see above; return value - flist = [] # holds the features in order - # list empty -> no features to parse - # list used as a stack - # last element = top of stack; - # and the element we currently - # collecting source-code lines for - fouter = [] # holds the xml-nodes of the ifdefs/endifs - # in order like flist - fcode = [] # holds the code of the features in - # order like flist - finner = [] # holds the tags of the features in - # order like flist - condinhist = [] # order of the conditional includes - # with feature names - parcon = False # parse-conditional-flag - parend = False # parse-endif-flag - _ = 0 # else and elif depth + features = OrderedDict({}) # see above; return value + featuresgrinner = [] # see above; return value + featuresgrouter = [] # see above; return value + flist = [] # holds the features in order + # list empty -> no features to parse + # list used as a stack + # last element = top of stack; + # and the element we currently + # collecting source-code lines for + fouter = [] # holds the xml-nodes of the ifdefs/endifs + # in order like flist + fcode = [] # holds the code of the features in + # order like flist + finner = [] # holds the tags of the features in + # order like flist + condinhist = [] # order of the conditional includes + # with feature names + parcon = False # parse-conditional-flag + parend = False # parse-endif-flag + _ = 0 # else and elif depth elses = [] ifdef_number = 0 @@ -481,20 +476,19 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # hitting on conditional-macro if ((tag in __conditionals_all) and (event == 'start') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = True # hitting on conditional-macro else or elif if (((tag in __conditionals_else) or (tag in __conditionals_elif)) and (event == 'start') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace ifdef_number += 1 - # hitting next conditional macro; any of ifdef, else or elif if ((tag in __conditionals_all) and (event == 'end') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = False # with else or elif we finish up the last if, therefor @@ -502,15 +496,18 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): if ((tag in __conditionals_else) or (tag in __conditionals_elif)): (features, featuresgrinner, - fcode, flist, finner) = _wrapFeatureUp(features, - featuresgrinner, fcode, flist, finner) + fcode, flist, finner) = _wrapFeatureUp(features, + featuresgrinner, fcode, flist, finner) fname = _getMacroSignature(elem) - if fname: condinhist.append((tag, fname)) - else: condinhist.append((tag, '')) + if fname: + condinhist.append((tag, fname)) + else: + condinhist.append((tag, '')) fsig = _getFeatureSignature(condinhist, options) - if (tag in __conditionals): fouter.append([]) + if tag in __conditionals: + fouter.append([]) fouter[-1] += ([(fsig, elem)]) flist.append(fsig) fcode.append('') @@ -523,35 +520,29 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): parcon = False # hitting end-tag of define-macro - if ((tag in __macro_define) \ - and (event == 'end') \ - and (ns == __cppnscpp)): + if (tag in __macro_define) and (event == 'end') and (ns == __cppnscpp): _parseAndAddDefine(elem) - # iterateting in subtree of conditional-node + # iterating in subtree of conditional-node if parcon: continue # handling endif-macro # hitting an endif-macro start-tag - if ((tag in __conditionals_endif) \ - and (event == "start") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "start") and (ns == __cppnscpp): # check the cpp:namespace parend = True # hitting the endif-macro end-tag - if ((tag in __conditionals_endif) \ - and (event == "end") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "end") and (ns == __cppnscpp): # check the cpp:namespace parend = False (features, featuresgrinner, fcode, flist, finner) = \ _wrapFeatureUp(features, featuresgrinner, - fcode, flist, finner) + fcode, flist, finner) (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, - featuresgrouter, elem) + featuresgrouter, elem) - while (condinhist[-1][0] != 'if'): + while condinhist[-1][0] != 'if': if condinhist[-1][0] == 'else': elses.append(ifdef_number) condinhist = condinhist[:-1] @@ -564,22 +555,24 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): continue # collect the source-code of the feature - if (len(flist)): - if ((event == "start") and (elem.text)): + if len(flist): + if (event == "start") and elem.text: fcode[-1] += elem.text - if ((event == "end") and (elem.tail)): + if (event == "end") and elem.tail: fcode[-1] += elem.tail - if (ns == __cppnsdef or tag not in __conditionals_all): + if ns == __cppnsdef or tag not in __conditionals_all: finner[-1].append((tag, event, elem.sourceline)) - if (flist): + if flist: raise IfdefEndifMismatchError() - return (features, featuresgrinner, featuresgrouter, elses) + return features, featuresgrinner, featuresgrouter, elses __nestedIfdefsLevels = [] __nestingDepthsOfBranches = [] + + def _getNestingDepths(root): """This function counts the number of nested ifdefs (conditionals) within the source-file in two different ways. @@ -607,7 +600,7 @@ def _getNestingDepths(root): # if a branch ends somehow if ((tag in __conditionals_ending) - and (ns == __cppnscpp)): + and (ns == __cppnscpp)): # reduce nesting level cncur -= 1 @@ -629,13 +622,13 @@ def _getNestingDepths(root): # if hitting the next conditional if ((tag in __conditionals_all) - and (ns == __cppnscpp)): + and (ns == __cppnscpp)): # increase nesting level cncur += 1 # gather the nesting depth of each #ifdef block (incl. #else/#elif branches) - if (tag in __conditionals): + if tag in __conditionals: cnlist.append(cncur) # add top-level signatures to history @@ -644,7 +637,7 @@ def _getNestingDepths(root): # if #else is reached, its empty signature must be rewritten as # negation of previous signatures within this #ifdef block if tag in __conditionals_else: - #FIXME how to do this if rewriting is enabled?! + # FIXME how to do this if rewriting is enabled?! newsig = ['!(' + xsig + ')' for (_, _, xsig, _) in sigblockhist] sigblockhist.append((__curfile, elem, " && ".join(newsig), -1)) @@ -659,7 +652,7 @@ def _getNestingDepths(root): # # DEBUG # print "%s %s: %s (max: %s)" % (tag, _getMacroSignature(elem), cncur, cnmax) - if (len(cnlist) > 0): + if 0 < len(cnlist): nnitmp = filter(lambda n: n > 0, cnlist) __nestedIfdefsLevels += nnitmp @@ -671,17 +664,18 @@ def _getScatteringTanglingValues(sigs, defines): defines according to the given mapping of a define to occurances in the signatures. The input is all feature-signatures and all defines.""" - #TODO insert tuples into description! + + # TODO insert tuples into description! def __add(x, y): """This method is a helper function to add values of lists pairwise. See below for more information.""" - return x+y + return x + y - scat = list() # relation define to signatures - tang = [0]*len(sigs) # signatures overall + scat = list() # relation define to signatures + tang = [0] * len(sigs) # signatures overall for d in defines: - dre = re.compile(r'\b'+d+r'\b') # using word boundaries + dre = re.compile(r'\b' + d + r'\b') # using word boundaries vec = map(lambda s: not dre.search(s) is None, sigs) scat.append(vec.count(True)) tang = map(__add, tang, vec) @@ -691,7 +685,7 @@ def __add(x, y): scatdict = zip(defines, scat) tangdict = zip(sigs, tang) - return (scatdict, tangdict) + return scatdict, tangdict def _checkForEquivalentSig(l, sig): @@ -704,7 +698,7 @@ def _checkSigEquivalence(sig1, sig2): It uses an xmlrpc-call on troi.fim.uni-passau.de.""" global __errorfexp global __errormatch - if not (sig1 and sig2): # ommit empty signatures + if not (sig1 and sig2): # ommit empty signatures return False # if options.str: @@ -717,26 +711,25 @@ def _checkSigEquivalence(sig1, sig2): raise NoEquivalentSigError() -def resetModule() : +def resetModule(): global __macrofuncs, __defset, __defsetf, __nestedIfdefsLevels, __nestingDepthsOfBranches - __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" - __defset = set() # macro-objects - __defsetf = dict() # macro-objects per file + __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", + # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" + __defset = set() # macro-objects + __defsetf = dict() # macro-objects per file __nestedIfdefsLevels = [] __nestingDepthsOfBranches = [] def apply(folder, options): - """This function applies the analysis to all xml-files in that directory and take the results and joins them together. Results are getting written into the fdcsv-file.""" # overall status variables resetModule() - sigmap = {} # {: []} - afeatures = {} # identified features; {: (depth, [code])} + sigmap = {} # {: []} + afeatures = {} # identified features; {: (depth, [code])} def _mergeFeatures(ffeatures): """This function merges the, with the parameter given @@ -747,9 +740,9 @@ def _mergeFeatures(ffeatures): try: sigmatch = _checkForEquivalentSig(sigmap.keys(), psig) (tmpdepth, tmpcode) = afeatures[sigmap[sigmatch][0]] -# if (tmpdepth != depth): -# print("INFO: depths of feature fragments do not" + -# " match (%s, %s)!" % (str(tmpdepth), str(depth))) + # if (tmpdepth != depth): + # print("INFO: depths of feature fragments do not" + + # " match (%s, %s)!" % (str(tmpdepth), str(depth))) tmpdepth = min(tmpdepth, depth) tmpcode += code afeatures[sigmap[sigmatch][0]] = (tmpdepth, tmpcode) @@ -760,7 +753,6 @@ def _mergeFeatures(ffeatures): afeatures[sig] = (depth, list(code)) sigmap[psig] = [sig] - global __curfile fcount = 0 files = returnFileNames(folder, ['.xml']) @@ -807,7 +799,7 @@ def _mergeFeatures(ffeatures): # preparation: opn file for writing stfheadings = ['name', 'values'] - stfrow = [None]*len(stfheadings) + stfrow = [None] * len(stfheadings) stfhandle, stfwriter = _prologCSV(os.path.join(folder, os.pardir), __metricvaluesfile, stfheadings) # scattering and tangling values @@ -842,30 +834,33 @@ def _mergeFeatures(ffeatures): # each signature is used only once per project (string equality) (scatvalues_merged, tangvalues_merged) = _getScatteringTanglingValues(list(set(sigs)), defs) - sd, sdcsv = _prologCSV(os.path.join(folder, os.pardir), "merged_scattering_degrees.csv", ["define","SD"], delimiter=",") + sd, sdcsv = _prologCSV(os.path.join(folder, os.pardir), "merged_scattering_degrees.csv", ["define", "SD"], + delimiter=",") for (define, scat) in scatvalues_merged: - sdcsv.writerow([define,scat]) + sdcsv.writerow([define, scat]) sd.close() - td, tdcsv = _prologCSV(os.path.join(folder, os.pardir), "merged_tangling_degrees.csv", ["signature","TD"], delimiter=",") + td, tdcsv = _prologCSV(os.path.join(folder, os.pardir), "merged_tangling_degrees.csv", ["signature", "TD"], + delimiter=",") for (sig, tang) in tangvalues_merged: - tdcsv.writerow([sig,tang]) + tdcsv.writerow([sig, tang]) td.close() - nd, ndcsv = _prologCSV(os.path.join(folder, os.pardir), "nesting_degrees_toplevel_branches.csv", ["file", "signature", "ND"], delimiter=",") # , "linenumber" + nd, ndcsv = _prologCSV(os.path.join(folder, os.pardir), "nesting_degrees_toplevel_branches.csv", + ["file", "signature", "ND"], delimiter=",") # , "linenumber" for (file, elem, sig, depth) in __nestingDepthsOfBranches: - #adjust file name if wanted - if options.filenamesRelative : # relative file name (root is project folder (not included in path)) + # adjust file name if wanted + if options.filenamesRelative: # relative file name (root is project folder (not included in path)) file = os.path.relpath(file, folder) - if options.filenames == options.FILENAME_SRCML : # cppstats file names - pass # nothing to do here, as the file path is the cppstats path by default - if options.filenames == options.FILENAME_SOURCE : # source file name + if options.filenames == options.FILENAME_SRCML: # cppstats file names + pass # nothing to do here, as the file path is the cppstats path by default + if options.filenames == options.FILENAME_SOURCE: # source file name file = file.replace(".xml", "").replace("/_cppstats/", "/source/", 1) # print information to file - ndcsv.writerow([file, sig, depth]) # , elem.sourceline - 1 + ndcsv.writerow([file, sig, depth]) # , elem.sourceline - 1 nd.close() @@ -873,12 +868,12 @@ def _mergeFeatures(ffeatures): # add command line options def addCommandLineOptionsMain(optionparser): - ''' add command line options for a direct call of this script''' + """ add command line options for a direct call of this script""" optionparser.add_argument("--folder", dest="folder", - help="input folder [default=%(default)s]", default=".") + help="input folder [default=%(default)s]", default=".") -def addCommandLineOptions(optionparser) : +def addCommandLineOptions(optionparser): # TODO implement CSP solving? # optionparser.add_option("--csp", dest="csp", action="store_true", # default=False, help="make use of csp solver to check " \ @@ -893,7 +888,7 @@ def addCommandLineOptions(optionparser) : "(exception are #else tags, which ARE rewritten as " "negation of the #if branch! see also --norewriteelse " "of analysis GENERALVALUES)") - #FIXME add command line function to remove #else too! + # FIXME add command line function to remove #else too! # ################################################ @@ -906,7 +901,6 @@ def getResultsFile(): ################################################## if __name__ == '__main__': - ################################################## # options parsing parser = ArgumentParser(formatter_class=RawTextHelpFormatter) @@ -919,7 +913,7 @@ def getResultsFile(): # main folder = os.path.abspath(options.folder) - if (os.path.isdir(folder)): + if os.path.isdir(folder): apply(folder, options) else: sys.exit(-1) From 8a7fd76b6cda8be56e86c399fab9f59be6a687c5 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 11:43:26 +0100 Subject: [PATCH 06/72] Fix analyses/interaction.py --- analyses/generalvalues.py | 2 - analyses/interaction.py | 567 +++++++++++++++++++------------------- 2 files changed, 285 insertions(+), 284 deletions(-) diff --git a/analyses/generalvalues.py b/analyses/generalvalues.py index 7e37aa3..3d4f688 100644 --- a/analyses/generalvalues.py +++ b/analyses/generalvalues.py @@ -41,8 +41,6 @@ # python-lxml module from lxml import etree -# statistics module -import numpy as np # pyparsing module import pyparsing as pypa diff --git a/analyses/interaction.py b/analyses/interaction.py index a000c02..d4dbfe4 100644 --- a/analyses/interaction.py +++ b/analyses/interaction.py @@ -29,7 +29,6 @@ import os import re import sys -import xmlrpclib from argparse import ArgumentParser, RawTextHelpFormatter # ################################################# @@ -38,21 +37,22 @@ __lib_subfolder = "lib" sys.path.append(os.path.abspath(__lib_subfolder)) # lib subfolder - # ################################################# # external modules # enums from enum import Enum - # python-lxml module +# python-lxml module from lxml import etree - # statistics module -from statlib import pstat +# statistics module +import numpy as np # pyparsing module import pyparsing as pypa -pypa.ParserElement.enablePackrat() # speed up parsing -sys.setrecursionlimit(8000) # handle larger expressions +pypa.ParserElement.enablePackrat() # speed up parsing +sys.setrecursionlimit(8000) # handle larger expressions + +from functools import reduce ################################################## # config: @@ -77,31 +77,32 @@ __conditionals_else = ['else'] __conditionals_endif = ['endif'] __conditionals_all = __conditionals + __conditionals_elif + \ - __conditionals_else + __conditionals_else __macro_define = ['define'] -__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" -__curfile = '' # current processed xml-file -__defset = set() # macro-objects -__defsetf = dict() # macro-objects per file +__macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", +# used as "GLIBVERSION(x,y,z) 100*x+10*y+z" +__curfile = '' # current processed xml-file +__defset = set() # macro-objects +__defsetf = dict() # macro-objects per file + # collected statistics class __statsorder(Enum): - FILENAME = 0 # name of the file - NUMANNOTATIONS = 1 # number of all annotations - NOIANNOTATIONS = 2 # number of interacting annotations - # example: A&B&C&D --> A&B, B&C, B&D - # example: A&B&C&D -/> A&B, B&C, A&C + FILENAME = 0 # name of the file + NUMANNOTATIONS = 1 # number of all annotations + NOIANNOTATIONS = 2 # number of interacting annotations + # example: A&B&C&D --> A&B, B&C, B&D + # example: A&B&C&D -/> A&B, B&C, A&C -################################################## +################################################## ################################################## # helper functions, constants and errors -def returnFileNames(folder, extfilt = ['.xml']): - '''This function returns all files of the input folder - and its subfolders.''' +def returnFileNames(folder, extfilt=['.xml']): + """This function returns all files of the input folder + and its subfolders.""" filesfound = list() if os.path.isdir(folder): @@ -111,25 +112,22 @@ def returnFileNames(folder, extfilt = ['.xml']): currentfolder = wqueue[0] wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) - tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) - tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) - tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles = list(filter(lambda n: os.path.isfile(os.path.join(currentfolder, n)), foldercontent)) + tmpfiles = list(filter(lambda n: os.path.splitext(n)[1] in extfilt, tmpfiles)) + tmpfiles = list(map(lambda n: os.path.join(currentfolder, n), tmpfiles)) filesfound += tmpfiles - tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) - tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders = list(filter(lambda n: os.path.isdir(os.path.join(currentfolder, n)), foldercontent)) + tmpfolders = list(map(lambda n: os.path.join(currentfolder, n), tmpfolders)) wqueue += tmpfolders return filesfound + def uniqueItems(l): l = sorted(l) return list(k for k, _ in itertools.groupby(l)) + def _flatten(l): """This function takes a list as input and returns a flatten version of the list. So all nested lists are unpacked and moved up to the @@ -142,7 +140,7 @@ def _flatten(l): i -= 1 break else: - l[i:i+1] = l[i] + l[i:i + 1] = l[i] i += 1 return l @@ -154,10 +152,10 @@ def _collectDefines(d): but not #define GLIBCVER(x,y,z) ... """ __defset.add(d[0]) - if __defsetf.has_key(__curfile): + if __curfile in __defsetf: __defsetf[__curfile].add(d[0]) else: - __defsetf[__curfile] = set([d[0]]) + __defsetf[__curfile] = {d[0]} return d @@ -173,41 +171,41 @@ def _collectDefines(d): __string = pypa.QuotedString('\'', '\\') __hexadec = \ - pypa.Literal('0x').suppress() + \ - pypa.Word(pypa.hexnums).\ - setParseAction(lambda t: str(int(t[0], 16))) + \ - pypa.Optional(__numlitu) + \ - pypa.Optional(__numlitl) + \ - pypa.Optional(__numlitl) + pypa.Literal('0x').suppress() + \ + pypa.Word(pypa.hexnums).setParseAction(lambda t: str(int(t[0], 16))) + \ + pypa.Optional(__numlitu) + \ + pypa.Optional(__numlitl) + \ + pypa.Optional(__numlitl) __integer = \ - pypa.Optional('~') + \ - pypa.Word(pypa.nums+'-') + \ - pypa.Optional(__numlitu) + \ - pypa.Optional(__numlitl) + \ - pypa.Optional(__numlitl) - -__identifier = \ - pypa.Word(pypa.alphanums+'_'+'-'+'@'+'$') # @ not allowed but they do occur -__arg = pypa.Word(pypa.alphanums+'_') -__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + \ - __arg) + pypa.Optional('~') + \ + pypa.Word(pypa.nums + '-') + \ + pypa.Optional(__numlitu) + \ + pypa.Optional(__numlitl) + \ + pypa.Optional(__numlitl) + +__identifier = pypa.Word(pypa.alphanums + '_' + '-' + '@' + '$') # @ not allowed but they do occur +__arg = pypa.Word(pypa.alphanums + '_') +__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + __arg) __fname = pypa.Word(pypa.alphas + '_' + '#', pypa.alphanums + '_' + '#') -__function = pypa.Group(__fname + pypa.Literal('(').suppress() + \ - __args + pypa.Literal(')').suppress()) +__function = pypa.Group(__fname + pypa.Literal('(').suppress() + __args + pypa.Literal(')').suppress()) class NoEquivalentSigError(Exception): def __init__(self): pass + def __str__(self): - return ("No equivalent signature found!") + return "No equivalent signature found!" + class IfdefEndifMismatchError(Exception): def __init__(self): pass + def __str__(self): - return ("Ifdef and endif do not match!") + return "Ifdef and endif do not match!" + ################################################## @@ -222,22 +220,25 @@ def _collapseSubElementsToList(node): return ''.join([it for it in itdesc]) - def _parseFeatureSignature(sig): """This function parses a given feature-signature.""" mal = set() - def _rewriteOne(p): return '' - def _rewriteTwo(p): return '' - def _addIdentifier2Mal(p): mal.add(p[0]) + def _rewriteOne(p): + return '' - operand = __string | __hexadec | __function | __integer | \ - __identifier.setParseAction(_addIdentifier2Mal) + def _rewriteTwo(p): + return '' + + def _addIdentifier2Mal(p): + mal.add(p[0]) + + operand = __string | __hexadec | __function | __integer | __identifier.setParseAction(_addIdentifier2Mal) compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / % & | << >>') expr = pypa.operatorPrecedence(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), - ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), + ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), (compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), ('&&', 2, pypa.opAssoc.LEFT, _rewriteTwo), @@ -246,14 +247,13 @@ def _addIdentifier2Mal(p): mal.add(p[0]) try: rsig = expr.parseString(sig)[0] - except pypa.ParseException, e: - print('ERROR (parse): cannot parse sig (%s) -- (%s)' % - (sig, e.col)) + except pypa.ParseException as e: + print(f'ERROR (parse): cannot parse sig ({sig}) -- ({e.col})') return sig except RuntimeError: - print('ERROR (time): cannot parse sig (%s)' % (sig)) + print(f'ERROR (time): cannot parse sig ({sig})') return sig - return (mal, ''.join(rsig)) + return mal, ''.join(rsig) def _getMacroSignature(ifdefnode): @@ -272,9 +272,9 @@ def _getMacroSignature(ifdefnode): # get either the expr or the name tag, # which is always the second descendant - if (tag in ['if', 'elif', 'ifdef', 'ifndef']): + if tag in ['if', 'elif', 'ifdef', 'ifndef']: nexpr = [itex for itex in ifdefnode.iterdescendants()] - if (len(nexpr) == 1): + if len(nexpr) == 1: res = nexpr[0].tail else: nexpr = nexpr[1] @@ -288,7 +288,7 @@ def _prologCSV(folder): fd = open(os.path.join(folder, __outputfile), 'w') fdcsv = csv.writer(fd, delimiter=',') fdcsv.writerow(__statsorder.__members__.keys()) - return (fd, fdcsv) + return fd, fdcsv def _countNestedIfdefs(root): @@ -300,23 +300,24 @@ def _countNestedIfdefs(root): for elem in elements: ns, tag = __cpprens.match(elem.tag).groups() - if ((tag in __conditionals_endif) - and (ns == __cppnscpp)): cncur -= 1 - if ((tag in __conditionals) - and (ns == __cppnscpp)): + if (tag in __conditionals_endif) and (ns == __cppnscpp): + cncur -= 1 + if (tag in __conditionals) and (ns == __cppnscpp): cncur += 1 cnlist.append(cncur) - if (len(cnlist) > 0): + if len(cnlist) > 0: nnimax = max(cnlist) - nnitmp = filter(lambda n: n > 0, cnlist) - nnimean = pstat.stats.lmean(nnitmp) + nnitmp = list(filter(lambda n: n > 0, cnlist)) + nnimean = np.mean(nnitmp) else: nnimax = 0 nnimean = 0 - if (len(cnlist) > 1): nnistd = pstat.stats.lstdev(cnlist) - else: nnistd = 0 - return (nnimax, nnimean, nnistd) + if len(cnlist) > 1: + nnistd = np.std(cnlist) + else: + nnistd = 0 + return nnimax, nnimean, nnistd def _getFeatureSignature(condinhist): @@ -326,7 +327,7 @@ def _getFeatureSignature(condinhist): # signature; reason is elements like else or elif, which mean # basically invert the fname found before # rewritelist = [(tag, fname, )] - rewritelist = [None]*len(condinhist) + rewritelist = [None] * len(condinhist) cur = -1 for tag, fname in condinhist: @@ -334,8 +335,8 @@ def _getFeatureSignature(condinhist): if tag == 'if': rewritelist[cur] = (tag, fname, False) if tag in ['elif', 'else']: - (t, f, _) = rewritelist[cur-1] - rewritelist[cur-1] = (t, f, True) + (t, f, _) = rewritelist[cur - 1] + rewritelist[cur - 1] = (t, f, True) rewritelist[cur] = (tag, fname, False) fsig = '' @@ -373,7 +374,7 @@ def _getASTFuture(node): node itself.""" dess = [] - while (node is not None): + while node is not None: dess += [sib for sib in node.itersiblings(preceding=False)] node = node.getparent() @@ -399,7 +400,7 @@ def _parseAndAddDefine(node): except pypa.ParseException: return - iden = ''.join(map(str, res[0])) + iden = ''.join(list(map(str, res[0]))) expn = res[-1] para = res[1:-1] __macrofuncs[iden] = (para, expn) @@ -423,58 +424,57 @@ def _getFeatures(root): """ def _wrapGrOuterUp(fouter, featuresgrouter, eelem): - itouter = fouter[-1] # feature surround tags + itouter = fouter[-1] # feature surround tags fouter = fouter[:-1] selem = itouter[0][1] for (sig, _) in itouter: featuresgrouter.append((sig, selem, eelem)) - return (fouter, featuresgrouter) - + return fouter, featuresgrouter def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # wrap up the feature - if (not flist): + if not flist: raise IfdefEndifMismatchError() - itsig = flist[-1] # feature signature + itsig = flist[-1] # feature signature flist = flist[:-1] - itcode = fcode[-1] # feature code + itcode = fcode[-1] # feature code itcode = itcode.replace('\n\n', '\n') - itcode = itcode[1:] # itcode starts with '\n'; del + itcode = itcode[1:] # itcode starts with '\n'; del fcode = fcode[:-1] - itinner = finner[-1] # feature enclosed tags + itinner = finner[-1] # feature enclosed tags finner = finner[:-1] # handle the feature code - if (features.has_key(itsig)): + if itsig in features: features[itsig][1].append(itcode) else: - features[itsig] = (len(flist)+1, [itcode]) + features[itsig] = (len(flist) + 1, [itcode]) # handle the inner granularity featuresgrinner.append((itsig, itinner)) - return (features, featuresgrinner, fcode, flist, finner) - - features = {} # see above; return value - featuresgrinner = [] # see above; return value - featuresgrouter = [] # see above; return value - flist = [] # holds the features in order - # list empty -> no features to parse - # list used as a stack - # last element = top of stack; - # and the element we currently - # collecting source-code lines for - fouter = [] # holds the xml-nodes of the ifdefs/endifs - # in order like flist - fcode = [] # holds the code of the features in - # order like flist - finner = [] # holds the tags of the features in - # order like flist - condinhist = [] # order of the conditional includes - # with feature names - parcon = False # parse-conditional-flag - parend = False # parse-endif-flag - _ = 0 # else and elif depth + return features, featuresgrinner, fcode, flist, finner + + features = {} # see above; return value + featuresgrinner = [] # see above; return value + featuresgrouter = [] # see above; return value + flist = [] # holds the features in order + # list empty -> no features to parse + # list used as a stack + # last element = top of stack; + # and the element we currently + # collecting source-code lines for + fouter = [] # holds the xml-nodes of the ifdefs/endifs + # in order like flist + fcode = [] # holds the code of the features in + # order like flist + finner = [] # holds the tags of the features in + # order like flist + condinhist = [] # order of the conditional includes + # with feature names + parcon = False # parse-conditional-flag + parend = False # parse-endif-flag + _ = 0 # else and elif depth # iterate over all tags separately - and -tag for event, elem in etree.iterwalk(root, events=("start", "end")): @@ -484,13 +484,13 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): # hitting on conditional-macro if ((tag in __conditionals_all) and (event == 'start') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = True # hitting next conditional macro; any of ifdef, else or elif if ((tag in __conditionals_all) and (event == 'end') - and (ns == __cppnscpp)): # check the cpp:namespace + and (ns == __cppnscpp)): # check the cpp:namespace parcon = False # with else or elif we finish up the last if, therefor @@ -498,15 +498,18 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): if ((tag in __conditionals_else) or (tag in __conditionals_elif)): (features, featuresgrinner, - fcode, flist, finner) = _wrapFeatureUp(features, - featuresgrinner, fcode, flist, finner) + fcode, flist, finner) = _wrapFeatureUp(features, + featuresgrinner, fcode, flist, finner) fname = _getMacroSignature(elem) - if fname: condinhist.append((tag, fname)) - else: condinhist.append((tag, '')) + if fname: + condinhist.append((tag, fname)) + else: + condinhist.append((tag, '')) fsig = _getFeatureSignature(condinhist) - if (tag in __conditionals): fouter.append([]) + if tag in __conditionals: + fouter.append([]) fouter[-1] += ([(fsig, elem)]) flist.append(fsig) fcode.append('') @@ -519,35 +522,27 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): parcon = False # hitting end-tag of define-macro - if ((tag in __macro_define) \ - and (event == 'end') \ - and (ns == __cppnscpp)): + if (tag in __macro_define) and (event == 'end') and (ns == __cppnscpp): _parseAndAddDefine(elem) - # iterateting in subtree of conditional-node + # iterating in subtree of conditional-node if parcon: continue # handling endif-macro # hitting an endif-macro start-tag - if ((tag in __conditionals_endif) \ - and (event == "start") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "start") and (ns == __cppnscpp): # check the cpp:namespace parend = True # hitting the endif-macro end-tag - if ((tag in __conditionals_endif) \ - and (event == "end") \ - and (ns == __cppnscpp)): # check the cpp:namespace + if (tag in __conditionals_endif) and (event == "end") and (ns == __cppnscpp): # check the cpp:namespace parend = False (features, featuresgrinner, fcode, flist, finner) = \ - _wrapFeatureUp(features, featuresgrinner, - fcode, flist, finner) - (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, - featuresgrouter, elem) + _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner) + (fouter, featuresgrouter) = _wrapGrOuterUp(fouter, featuresgrouter, elem) - while (condinhist[-1][0] != 'if'): + while condinhist[-1][0] != 'if': condinhist = condinhist[:-1] condinhist = condinhist[:-1] @@ -556,18 +551,18 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): continue # collect the source-code of the feature - if (len(flist)): - if ((event == "start") and (elem.text)): + if len(flist): + if (event == "start") and elem.text: fcode[-1] += elem.text - if ((event == "end") and (elem.tail)): + if (event == "end") and elem.tail: fcode[-1] += elem.tail - if (ns == __cppnsdef or tag not in __conditionals_all): + if ns == __cppnsdef or tag not in __conditionals_all: finner[-1].append((tag, event, elem.sourceline)) - if (flist): + if flist: raise IfdefEndifMismatchError() - return (features, featuresgrinner, featuresgrouter) + return features, featuresgrinner, featuresgrouter def _getOuterGranularity(fnodes): @@ -578,7 +573,7 @@ def _getOuterGranularity(fnodes): grouter = list() for (sig, selem, _) in fnodes: - tags = _getASTHistory(selem)[:-1] # cut of unit-tag + tags = _getASTHistory(selem)[:-1] # cut of unit-tag grouter.append((sig, tags, selem.sourceline)) return grouter @@ -594,93 +589,100 @@ def _getOuterGranularityStats(lgran): """ gotopbgr = 0 gofunbgr = 0 - gostrbrl = 0 # local - gostrbrg = 0 # global + gostrbrl = 0 # local + gostrbrg = 0 # global goinnbgr = 0 goexpbgr = 0 gostmbgr = 0 gopambgr = 0 - goerror = 0 + goerror = 0 for (_, gran, line) in lgran: if len(gran) == 0: gotopbgr += 1 continue if gran[0] in ['block']: - if len(gran) == 1: # configure the method signature + if len(gran) == 1: # configure the method signature gofunbgr += 1 continue if gran[1] in ['function', 'extern', 'block']: gofunbgr += 1 elif gran[1] in ['struct', 'union', - 'enum']: # test_struct_union_enum.c - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + 'enum']: # test_struct_union_enum.c + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 elif gran[1] in ['expr']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 elif gran[1] in ['while', 'for', 'then', 'do', - 'else', 'switch', 'case', 'default']: - goinnbgr += 1 # test_loop.c - elif gran[1] in ['decl']: # test_struct_union_enum.c - if 'function' in gran[3:]: gostrbrl += 1 - else: gostrbrg += 1 + 'else', 'switch', 'case', 'default']: + goinnbgr += 1 # test_loop.c + elif gran[1] in ['decl']: # test_struct_union_enum.c + if 'function' in gran[3:]: + gostrbrl += 1 + else: + gostrbrg += 1 else: - print('ERROR: gran (%s) at this ' - 'level unknown (line %s)' % (gran, line)) + print('ERROR: gran ({}) at this level unknown (line {})'.format(gran, line)) goerror += 1 continue elif gran[0] in ['expr']: - if gran[1] in ['expr_stmt']: # test_stmt.c + if gran[1] in ['expr_stmt']: # test_stmt.c gostmbgr += 1 - elif gran[1] in ['condition', 'return']: # test_condition.c + elif gran[1] in ['condition', 'return']: # test_condition.c goexpbgr += 1 - elif gran[1] in ['argument']: # test_call.c + elif gran[1] in ['argument']: # test_call.c gostmbgr += 1 elif gran[1] in ['block']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 - elif gran[1] in ['init', 'index']: # test_stmt.c + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 + elif gran[1] in ['init', 'index']: # test_stmt.c gostmbgr += 1 else: - print('ERROR: gran (%s) at this level' - 'unknown (line %s)' % (gran, line)) + print(f'ERROR: gran ({gran}) at this level unknown (line {line})') goerror += 1 continue - elif gran[0] in ['while', 'do']: # test_loop.c + elif gran[0] in ['while', 'do']: # test_loop.c goinnbgr += 1 continue elif gran[0] in ['expr_stmt'] and len(gran) == 1: gostmbgr += 1 continue elif gran[:3] == ['expr_stmt', 'block', 'struct']: - if 'function' in gran[2:]: gostrbrl += 1 - else: gostrbrg += 1 + if 'function' in gran[2:]: + gostrbrl += 1 + else: + gostrbrg += 1 continue - elif gran[0] in ['decl_stmt']: # test_stmt.c + elif gran[0] in ['decl_stmt']: # test_stmt.c gostmbgr += 1 continue - elif gran[0] in ['condition']: # test_condition.c + elif gran[0] in ['condition']: # test_condition.c goexpbgr += 1 continue elif gran[0] in ['if', 'else', 'case', 'default', - 'then', 'for']: # test_condition.c + 'then', 'for']: # test_condition.c goinnbgr += 1 continue elif gran[0] in ['parameter_list', - 'argument_list']: # test_call.c + 'argument_list']: # test_call.c gopambgr += 1 continue elif gran[0] in ['argument'] and gran[1] in ['argument_list']: gostmbgr += 1 - elif gran[0] in ['init'] and gran[1] in ['decl']: # test_stmt.c + elif gran[0] in ['init'] and gran[1] in ['decl']: # test_stmt.c gostmbgr += 1 continue - elif gran[0] in ['function']: # function prototype + elif gran[0] in ['function']: # function prototype continue else: - print('ERROR: outer granularity (%s, %s) not recognize!' % \ - (gran, line)) + print(f'ERROR: outer granularity ({gran}, {line}) not recognize!') goerror += 1 return (gotopbgr, gofunbgr, gostrbrl, gostrbrg, @@ -710,22 +712,20 @@ def _getInnerGranularityStats(igran): for (_, gran) in igran: for (tag, event, line) in gran: - if (skiptilltag != ''): + if skiptilltag != '': if (tag == skiptilltag[0] and event == 'end' and line == skiptilltag[2]): skiptilltag = '' continue - if tag in ['name', 'endif']: continue - elif tag in ['define', 'directive', 'include', - 'macro', 'undef']: + if tag in ['name', 'endif']: + continue + elif tag in ['define', 'directive', 'include', 'macro', 'undef']: gmacrogr += 1 - elif tag in ['struct', 'union', 'enum', 'function', 'extern', - 'function_decl', 'decl_stmt', 'typedef']: + elif tag in ['struct', 'union', 'enum', 'function', 'extern', 'function_decl', 'decl_stmt', 'typedef']: ginablgr += 1 - elif tag in ['if', 'while', 'return', 'then', - 'for', 'do', 'case', 'else', 'block']: + elif tag in ['if', 'while', 'return', 'then', 'for', 'do', 'case', 'else', 'block']: giunblgr += 1 elif tag in ['param', 'argument']: gipambgr += 1 @@ -736,13 +736,13 @@ def _getInnerGranularityStats(igran): elif tag in ['expr_stmt', 'decl', 'init']: gistmbgr += 1 else: - print('ERROR: inner granularity (%s, %s, %s)) ' - 'not recognized!' % (tag, event, line)) + print(f'ERROR: inner granularity ({tag}, {event}, {line})) not recognized!') gierror += 1 continue - if event == 'start': skiptilltag = (tag, event, line) + if event == 'start': + skiptilltag = (tag, event, line) - return (gmacrogr, ginablgr, giunblgr, giexpbgr, gistmbgr, gierror) + return gmacrogr, ginablgr, giunblgr, giexpbgr, gistmbgr, gierror def _getFeatureStats(features): @@ -758,30 +758,30 @@ def _getFeatureStats(features): """ lof = 0 nod = 0 - lofmin = -1 # feature-code can be empty + lofmin = -1 # feature-code can be empty lofmax = 0 lofmean = 0 lofstd = 0 nof = len(features.keys()) tmp = [item for (_, item) in features.itervalues()] tmp = _flatten(tmp) - floflist = map(lambda n: n.count('\n'), tmp) + floflist = list(map(lambda n: n.count('\n'), tmp)) - if (len(floflist)): + if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = reduce(lambda m,n: m+n, floflist) - lofmean = pstat.stats.lmean(floflist) + lof = reduce(lambda m, n: m + n, floflist) + lofmean = np.mean(floflist) - if (len(floflist) > 1): - lofstd = pstat.stats.lstdev(floflist) + if len(floflist) > 1: + lofstd = np.std(floflist) - return (nof, nod, lof, lofmin, lofmax, lofmean, lofstd) + return nof, nod, lof, lofmin, lofmax, lofmean, lofstd def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = filter(lambda (sig, (depth, code)): depth == 1, features.iteritems()) + nof1 = list(filter(lambda t: t[1][0] == 1, features.iteritems())) # t = (sig, (depth, code)) return nof1 @@ -809,13 +809,13 @@ def _compareFeatureCode(fcode): """ fcodes = set(fcode) - if (len(fcodes) == 1 and len(fcode) > 1): + if len(fcodes) == 1 and len(fcode) > 1: return "hom" - if (len(fcode) == len(fcodes)): + if len(fcode) == len(fcodes): return "het" - if (len(fcode) > len(fcodes)): + if len(fcode) > len(fcodes): return "hethom" scode = {} @@ -828,27 +828,27 @@ def _compareFeatureCode(fcode): for (key, (_, item)) in features.iteritems(): # distinguish according to feature-signature # shared code - if ('||' in key and (not '&&' in key)): + if '||' in key and ('&&' not in key): scode[key] = item # derivative only && - if ('&&' in key and (not '||' in key)): + if '&&' in key and ('||' not in key): deriv[key] = item # combination shared code and derivative - if ('&&' in key and '||' in key): + if '&&' in key and '||' in key: desc[key] = item # distinguish according to feature-code ret = _compareFeatureCode(item) - if (ret == "het"): + if ret == "het": het[key] = item - if (ret == "hom"): + if ret == "hom": hom[key] = item - if (ret == "hethom"): + if ret == "hethom": hethom[key] = item - return (scode, deriv, desc, het, hom, hethom) + return scode, deriv, desc, het, hom, hethom def _getNumOfDefines(defset): @@ -864,7 +864,7 @@ def _getNumOfDefines(defset): # basic operation of this function is to check __defset against # __macrofuncs funcmacros = __macrofuncs.keys() - funcmacros = map(lambda n: n.split('(')[0], funcmacros) + funcmacros = list(map(lambda n: n.split('(')[0], funcmacros)) funcmacros = set(funcmacros) return len((defset - funcmacros)) @@ -884,27 +884,35 @@ def _getScatteringTanglingDegrees(sigs, defines): def __add(x, y): """This method is a helper function to add values of lists pairwise. See below for more information.""" - return x+y + return x + y - scat = list() # relation define to signatures - tang = [0]*len(sigs) # signatures overall + scat = list() # relation define to signatures + tang = [0] * len(sigs) # signatures overall for d in defines: - dre = re.compile(r'\b'+d+r'\b') # using word boundaries - vec = map(lambda s: not dre.search(s) is None, sigs) + dre = re.compile(r'\b' + d + r'\b') # using word boundaries + vec = list(map(lambda s: not dre.search(s) is None, sigs)) scat.append(vec.count(True)) - tang = map(__add, tang, vec) + tang = list(map(__add, tang, vec)) - if (len(scat)): sdegmean = pstat.stats.lmean(scat) - else: sdegmean = 0 - if (len(scat) > 1): sdegstd = pstat.stats.lstdev(scat) - else: sdegstd = 0 + if len(scat): + sdegmean = np.mean(scat) + else: + sdegmean = 0 + if len(scat) > 1: + sdegstd = np.std(scat) + else: + sdegstd = 0 - if (len(tang)): tdegmean = pstat.stats.lmean(tang) - else: tdegmean = 0 - if (len(tang) > 1): tdegstd = pstat.stats.lstdev(tang) - else: tdegstd = 0 + if len(tang): + tdegmean = np.mean(tang) + else: + tdegmean = 0 + if len(tang) > 1: + tdegstd = np.std(tang) + else: + tdegstd = 0 - return (sdegmean, sdegstd, tdegmean, tdegstd) + return sdegmean, sdegstd, tdegmean, tdegstd def _getGranularityStats(fcodetags): @@ -916,20 +924,20 @@ def _getGranularityStats(fcodetags): expr -> expression """ _interestingtags = [ - 'define', # define statement - 'include', # include statement - 'decl_stmt', # declaration statement - 'expr_stmt', # expression statement - 'function_decl', # function declaration - 'parameter_list', # parameter list - 'param', # parameter + 'define', # define statement + 'include', # include statement + 'decl_stmt', # declaration statement + 'expr_stmt', # expression statement + 'function_decl', # function declaration + 'parameter_list', # parameter list + 'param', # parameter ] - _curskiptag = None # holds the element we are going to skip for - _skipedtags = [] # if we find one of the elements above, we - # start skipping the following tags, until - # the corresponding endtag is found; if we - # do not find the endtag, we are going to - # start analyzing the elements in _skipedtags + _curskiptag = None # holds the element we are going to skip for + _skipedtags = [] # if we find one of the elements above, we + # start skipping the following tags, until + # the corresponding endtag is found; if we + # do not find the endtag, we are going to + # start analyzing the elements in _skipedtags granstats = dict() for tag in _interestingtags: @@ -939,19 +947,19 @@ def _getGranularityStats(fcodetags): _curskiptag = None _skipedtags = [] for (tag, _, sourceline) in ftagsl: - if _curskiptag == None and tag in _interestingtags: + if _curskiptag is None and tag in _interestingtags: _curskiptag = tag continue - if _curskiptag != None and tag == _curskiptag: + if _curskiptag is not None and tag == _curskiptag: granstats[tag] = granstats[tag] + 1 _curskiptag = None _skipedtags = [] continue - if _curskiptag != None: + if _curskiptag is not None: _skipedtags.append((tag, sourceline)) continue -# if _skipedtags != []: -# print("ERROR: ", _skipedtags) + # if _skipedtags != []: + # print("ERROR: ", _skipedtags) return granstats @@ -965,7 +973,7 @@ def _checkSigEquivalence(sig1, sig2): It uses an xmlrpc-call on troi.fim.uni-passau.de.""" global __errorfexp global __errormatch - if not (sig1 and sig2): # ommit empty signatures + if not (sig1 and sig2): # ommit empty signatures return False # TODO csp vs str @@ -978,12 +986,13 @@ def _checkSigEquivalence(sig1, sig2): return it raise NoEquivalentSigError() -def resetModule() : + +def resetModule(): global __macrofuncs, __defset, __defsetf - __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", - # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" - __defset = set() # macro-objects - __defsetf = dict() # macro-objects per file + __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", + # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" + __defset = set() # macro-objects + __defsetf = dict() # macro-objects per file def apply(folder, options): @@ -993,8 +1002,8 @@ def apply(folder, options): # overall status variables resetModule() - sigmap = {} # {: []} - afeatures = {} # identified features; {: ([flag], depth, [code])} + sigmap = {} # {: []} + afeatures = {} # identified features; {: ([flag], depth, [code])} def _mergeFeatures(ffeatures): """This function merges the, with the parameter given @@ -1023,7 +1032,7 @@ def _mergeFeatures(ffeatures): global __curfile fcount = 0 files = returnFileNames(folder, ['.xml']) - fstats = [None]*len(__statsorder) + fstats = [None] * len(__statsorder) ftotal = len(files) # get statistics for all files; write results into csv @@ -1034,44 +1043,39 @@ def _mergeFeatures(ffeatures): try: tree = etree.parse(file) except etree.XMLSyntaxError: - print("ERROR: cannot parse (%s). Skipping this file." % - os.path.join(folder, file)) + print(f"ERROR: cannot parse ({os.path.join(folder, file)}). Skipping this file.") continue - print('INFO: parsing file (%5d) of (%5d) -- (%s).' % - (fcount, ftotal, os.path.join(folder, file))) + print(f'INFO: parsing file ({fcount:5d}) of ({ftotal:5d}) -- ({os.path.join(folder, file)}).') root = tree.getroot() try: (features, _, featuresgrouter) = _getFeatures(root) except IfdefEndifMismatchError: - print("ERROR: ifdef-endif mismatch in file (%s)" % - (os.path.join(folder, file))) + print(f"ERROR: ifdef-endif mismatch in file ({os.path.join(folder, file)})") continue _mergeFeatures(features) # filter annotations that do not have any c-code # filter annotations with less than 3 features - afeatureitems = filter(lambda (a, (f, d, c)): - c != [''], afeatures.items()) - annotations = map(lambda (a, (flag, b, c)): flag, afeatureitems) - annotations3andmore = filter(lambda a: len(a) > 2, annotations) + afeatureitems = list(filter(lambda t: t[1][2] != [''], afeatures.items())) # t = (a, (f, d, c)) + annotations = list(map(lambda t: t[1][0], afeatureitems)) # t = (a, (flag, b, c)) + annotations3andmore = list(filter(lambda a: len(a) > 2, annotations)) annotations3andmore = uniqueItems(annotations3andmore) - annotations3andmore = map(lambda s: set(s), annotations3andmore) + annotations3andmore = list(map(lambda s: set(s), annotations3andmore)) relevantannotations = list() missingannotations = list() noneannotations = list() # create all pairwise combinations of features for annotation in annotations3andmore: - combinations = map(lambda s: - set(s), list(itertools.combinations(annotation, 2))) + combinations = list(map(lambda s: set(s), list(itertools.combinations(annotation, 2)))) allcomb = len(combinations) - occcomblist = list(set()) + occcomblist = list(set()) # TODO: = [] ? for combination in combinations: if combination in annotations: occcomblist.append(combination) - combfeatset = reduce(set.union, occcomblist, set()) + combfeatset = set(reduce(set.union, occcomblist, set())) if combfeatset.issuperset(annotation): relevantannotations.append((annotation, occcomblist)) else: @@ -1089,11 +1093,12 @@ def _mergeFeatures(ffeatures): __outputfile ), 'w') - for i in relevantannotations: fd.write(str(i)+"\n") - fd.write("total annotations: %5d\n" % len(annotations3andmore)) - fd.write("relevant pairwise annotations: %5d\n" % len(relevantannotations)) - fd.write("missing pairwise annotations: %5d\n" % len(missingannotations)) - fd.write("none pairwise annotations: %5d\n" % len(noneannotations)) + for i in relevantannotations: + fd.write(str(i) + "\n") + fd.write(f"total annotations: {len(annotations3andmore):5d}\n") + fd.write(f"relevant pairwise annotations: {len(relevantannotations):5d}\n") + fd.write(f"missing pairwise annotations: {len(missingannotations):5d}\n") + fd.write(f"none pairwise annotations: {len(noneannotations):5d}\n") fd.close() @@ -1101,12 +1106,12 @@ def _mergeFeatures(ffeatures): # add command line options def addCommandLineOptionsMain(optionparser): - ''' add command line options for a direct call of this script''' + """ add command line options for a direct call of this script""" optionparser.add_argument("--folder", dest="folder", - help="input folder [default=.]", default=".") + help="input folder [default=.]", default=".") -def addCommandLineOptions(optionparser) : +def addCommandLineOptions(optionparser): # TODO implement CSP solving? # optionparser.add_argument("--csp", dest="csp", action="store_true", # default=False, help="make use of csp solver to check " \ @@ -1127,7 +1132,6 @@ def getResultsFile(): ################################################## if __name__ == '__main__': - ################################################## # options parsing parser = ArgumentParser(formatter_class=RawTextHelpFormatter) @@ -1140,8 +1144,7 @@ def getResultsFile(): # main folder = os.path.abspath(options.folder) - if (os.path.isdir(folder)): + if os.path.isdir(folder): apply(folder) else: sys.exit(-1) - From 9280f3df815d876853e6e9abfe1661944039ed7b Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 11:49:52 +0100 Subject: [PATCH 07/72] Make results of reduce a list or set --- analyses/derivative.py | 4 ++-- analyses/general.py | 2 +- analyses/interaction.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/analyses/derivative.py b/analyses/derivative.py index b542e10..e3b2b73 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -781,7 +781,7 @@ def _getFeatureStats(features): if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = reduce(lambda m, n: m + n, floflist) + lof = list(reduce(lambda m, n: m + n, floflist)) lofmean = np.mean(floflist) if len(floflist) > 1: @@ -1091,7 +1091,7 @@ def _mergeFeatures(ffeatures): projectpath = os.path.dirname(folder) fd = open(os.path.join(projectpath, __outputfile), 'w') - featurenames = reduce(set.union, annotations2andmore, set([])) + featurenames = set(reduce(set.union, annotations2andmore, set([]))) for i in featurenames: fd.write(i + '\n') for a, f in annotationmap.items(): diff --git a/analyses/general.py b/analyses/general.py index f34bd68..24a8374 100644 --- a/analyses/general.py +++ b/analyses/general.py @@ -928,7 +928,7 @@ def _getFeatureStats(features): if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = reduce(lambda m, n: m + n, floflist) + lof = list(reduce(lambda m, n: m + n, floflist)) lofmean = np.mean(floflist) if len(floflist) > 1: diff --git a/analyses/interaction.py b/analyses/interaction.py index d4dbfe4..34d64ad 100644 --- a/analyses/interaction.py +++ b/analyses/interaction.py @@ -770,7 +770,7 @@ def _getFeatureStats(features): if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = reduce(lambda m, n: m + n, floflist) + lof = list(reduce(lambda m, n: m + n, floflist)) lofmean = np.mean(floflist) if len(floflist) > 1: From c6d180558690224c129e5fb5eeb921fc515448b2 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 11:59:35 +0100 Subject: [PATCH 08/72] Fix cppstats/analysis.py --- cppstats/analysis.py | 76 ++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 42 deletions(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index e09bb2b..4e0374d 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -29,28 +29,21 @@ import os import sys import shutil # for copying files and folders -import errno # for error/exception handling -import threading # for parallelism -import subprocess # for calling other commands -import re # for regular expressions from abc import ABCMeta, abstractmethod # abstract classes -from argparse import ArgumentParser, RawTextHelpFormatter # for parameters to this script from collections import OrderedDict # for ordered dictionaries # ################################################# # imports from subfolders -import cppstats, cli +import cli # import different kinds of analyses from analyses import general, generalvalues, discipline, featurelocations, derivative, interaction - # ################################################# # global constants - # ################################################# # platform specific preliminaries @@ -58,19 +51,19 @@ __platform = sys.platform.lower() __iscygwin = False -if (__platform.startswith("cygwin")): +if __platform.startswith("cygwin"): __iscygwin = True -elif (__platform.startswith("darwin") or __platform.startswith("linux")): +elif __platform.startswith("darwin") or __platform.startswith("linux"): pass else: - print "Your system '" + __platform + "' is not supported right now." + print(f"Your system \'{__platform}\' is not supported right now.") # ################################################# # helper functions def notify(message): - if (__iscygwin): + if __iscygwin: return # FIXME enable notifications again! @@ -85,19 +78,19 @@ def notify(message): # abstract analysis thread class AbstractAnalysisThread(object): - '''This class analyzes a whole project according to the given kind of analysis in an independent thread.''' + """This class analyzes a whole project according to the given kind of analysis in an independent thread.""" __metaclass__ = ABCMeta def __init__(self, options, inputfolder=None, inputfile=None): self.options = options self.notrunnable = False - if (inputfolder): + if inputfolder: self.file = None self.folder = os.path.join(inputfolder, self.getPreparationFolder()) self.project = os.path.basename(self.folder) - elif (inputfile): + elif inputfile: self.file = inputfile self.outfile = self.options.outfile self.project = os.path.basename(self.file) @@ -114,34 +107,32 @@ def __init__(self, options, inputfolder=None, inputfile=None): else: self.notrunnable = True - def startup(self): # LOGGING - notify("starting '" + self.getName() + "' analysis:\n " + self.project) - print "# starting '" + self.getName() + "' analysis: " + self.project + notify(f"starting '{self.getName()}' analysis:\n {self.project}") + print(f"# starting '{self.getName()}' analysis: {self.project}") def teardown(self): # delete temp folder for file-based preparation - if (self.file): + if self.file: shutil.rmtree(self.tmpfolder) # LOGGING - notify("finished '" + self.getName() + "' analysis:\n " + self.project) - print "# finished '" + self.getName() + "' analysis: " + self.project + notify(f"finished '{self.getName()}' analysis:\n{self.project}") + print(f"# finished '{self.getName()}' analysis: {self.project}") def run(self): - - if (self.notrunnable): - print "ERROR: No single file or input list of projects given!" + if self.notrunnable: + print("ERROR: No single file or input list of projects given!") return self.startup() # copy srcml inputfile to tmp folder again and analyze project there! - if (self.file): + if self.file: currentFile = os.path.join(self.folder, self.project) - if (not currentFile.endswith(".xml")): + if not currentFile.endswith(".xml"): currentFile += ".xml" shutil.copyfile(self.file, currentFile) @@ -149,7 +140,7 @@ def run(self): self.analyze(self.folder) # copy main results file from tmp folder to destination, if given - if (self.file and self.resultsfile != self.outfile): + if self.file and self.resultsfile != self.outfile: shutil.copyfile(self.resultsfile, self.outfile) self.teardown() @@ -175,7 +166,7 @@ def addCommandLineOptions(cls, optionParser): pass @abstractmethod - def analyze(self): + def analyze(self, folder): pass @@ -330,9 +321,9 @@ def analyze(self, folder): __analysiskinds.append(entry) # exit, if there are no analysis threads available -if (len(__analysiskinds) == 0): - print "ERROR: No analysis tasks found! Revert your changes or call the maintainer." - print "Exiting now..." +if len(__analysiskinds) == 0: + print("ERROR: No analysis tasks found! Revert your changes or call the maintainer.") + print("Exiting now...") sys.exit(1) __analysiskinds = OrderedDict(__analysiskinds) @@ -356,7 +347,7 @@ def applyFile(kind, inputfile, options): def getFoldersFromInputListFile(inputlist): - ''' This method reads the given inputfile line-wise and returns the read lines without line breaks.''' + """ This method reads the given inputfile line-wise and returns the read lines without line breaks.""" file = open(inputlist, 'r') # open input file folders = file.read().splitlines() # read lines from file without line breaks @@ -364,9 +355,9 @@ def getFoldersFromInputListFile(inputlist): folders = filter(lambda f: not f.startswith("#"), folders) # remove commented lines folders = filter(os.path.isdir, folders) # remove all non-directories - folders = map(os.path.normpath, folders) # normalize paths for easier transformations + folders = map(os.path.normpath, folders) # normalize paths for easier transformations - #TODO log removed folders + # TODO log removed folders return folders @@ -404,36 +395,37 @@ def main(): # ################################################# # main - if (options.inputfile): + if options.inputfile: # split --file argument options.infile = os.path.normpath(os.path.abspath(options.inputfile[0])) # IN options.outfile = os.path.normpath(os.path.abspath(options.inputfile[1])) # OUT # check if inputfile exists - if (not os.path.isfile(options.infile)): - print "ERROR: input file '{}' cannot be found!".format(options.infile) + if not os.path.isfile(options.infile): + print(f"ERROR: input file '{options.infile}' cannot be found!") sys.exit(1) applyFile(options.kind, options.infile, options) - elif (options.inputlist): + elif options.inputlist: # handle --list argument options.inputlist = os.path.normpath(os.path.abspath(options.inputlist)) # LIST # check if list file exists - if (not os.path.isfile(options.inputlist)): - print "ERROR: input file '{}' cannot be found!".format(options.inputlist) + if not os.path.isfile(options.inputlist): + print(f"ERROR: input file '{options.inputlist}' cannot be found!") sys.exit(1) - if (options.allkinds): + if options.allkinds: applyFoldersAll(options.inputlist, options) else: applyFolders(options.kind, options.inputlist, options) else: - print "This should not happen! No input file or list of projects given!" + print("This should not happen! No input file or list of projects given!") sys.exit(1) + if __name__ == '__main__': main() From 8e5aba46825427080de17967618a5485837c3e30 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 12:06:01 +0100 Subject: [PATCH 09/72] Fix cppstats/cli.py --- cppstats/cli.py | 85 +++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 49 deletions(-) diff --git a/cppstats/cli.py b/cppstats/cli.py index 87ff3dc..5c5bbe1 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -25,14 +25,15 @@ # imports from the std-library import sys -from argparse import ArgumentParser, RawTextHelpFormatter, _VersionAction # for parameters to this script +from argparse import ArgumentParser, RawTextHelpFormatter # , _VersionAction # for parameters to this script # ################################################# # imports from subfolders -import preparation, analysis -import cppstats as cstats # import cppstats.py and avoid confusion with module - +# TODO Are those really not needed? +# import preparation +# import cppstats as cstats # import cppstats.py and avoid confusion with module +import analysis # ################################################# # external modules @@ -40,7 +41,6 @@ # enums from enum import Enum - # ################################################# # global constants @@ -58,30 +58,31 @@ class steps(Enum): # ################################################# # custom actions -class CppstatsVersionAction(_VersionAction): - # FIXME not needed for Python 3.4+! see http://bugs.python.org/issue18920 - ''' - subclass to _VersionAction as that class prints version information - to ``stderr``, but this subclass prints to ``stdout`` instead. - - Note: This is default in Python 3.4+. - ''' - - def __init__(self, *args, **kwargs): - """Initialisation method for the _VersionAction class""" - _VersionAction.__init__(self, *args, **kwargs) - - def __call__(self, parser, namespace, values, option_string=None): - version = self.version - if version is None: - version = parser.version - formatter = parser._get_formatter() - formatter.add_text(version) - # parser.exit(message=formatter.format_help()) - - # change output to sys.stdout and exit then without a message - parser._print_message(message=formatter.format_help(), file=sys.stdout) - parser.exit(status=0) +# TODO: _VersionAction does not seem to exist anymore +# class CppstatsVersionAction(_VersionAction): +# # FIXME not needed for Python 3.4+! see http://bugs.python.org/issue18920 +# ''' +# subclass to _VersionAction as that class prints version information +# to ``stderr``, but this subclass prints to ``stdout`` instead. +# +# Note: This is default in Python 3.4+. +# ''' +# +# def __init__(self, *args, **kwargs): +# """Initialisation method for the _VersionAction class""" +# _VersionAction.__init__(self, *args, **kwargs) +# +# def __call__(self, parser, namespace, values, option_string=None): +# version = self.version +# if version is None: +# version = parser.version +# formatter = parser._get_formatter() +# formatter.add_text(version) +# # parser.exit(message=formatter.format_help()) +# +# # change output to sys.stdout and exit then without a message +# parser._print_message(message=formatter.format_help(), file=sys.stdout) +# parser.exit(status=0) # ################################################# @@ -102,12 +103,11 @@ def getOptions(kinds, step=steps.ALL): parser = ArgumentParser(formatter_class=RawTextHelpFormatter) + # TODO see above # version (uses CppstatsVersionAction instead of 'version' as action) - parser.add_argument('--version', action=CppstatsVersionAction, version=cstats.version()) - + # parser.add_argument('--version', action=CppstatsVersionAction, version=cstats.version()) # ADD KIND ARGUMENT - # kinds kindgroup = parser.add_mutually_exclusive_group(required=False) kindgroup.add_argument("--kind", choices=kinds.keys(), dest="kind", @@ -116,9 +116,7 @@ def getOptions(kinds, step=steps.ALL): kindgroup.add_argument("-a", "--all", action="store_true", dest="allkinds", default=False, help="perform all available kinds of preparation/analysis [default: %(default)s]") - # ADD INPUT TYPE (list or file) - # input 1 inputgroup = parser.add_mutually_exclusive_group(required=False) # TODO check if True is possible some time... inputgroup.add_argument("--list", type=str, dest="inputlist", metavar="LIST", @@ -127,7 +125,8 @@ def getOptions(kinds, step=steps.ALL): # input 2 if step == steps.ALL: inputgroup.add_argument("--file", type=str, dest="inputfile", nargs=2, metavar=("IN", "OUT"), - help="a source file IN that is prepared and analyzed, the analysis results are written to OUT" + help="a source file IN that is prepared and analyzed, " + "the analysis results are written to OUT" "\n(--list is the default)") elif step == steps.PREPARATION: inputgroup.add_argument("--file", type=str, dest="inputfile", nargs=2, metavar=("IN", "OUT"), @@ -138,9 +137,7 @@ def getOptions(kinds, step=steps.ALL): help="a srcML file IN that is analyzed, the analysis results are written to OUT" "\n(--list is the default)") - # ADD VARIOUS STEP-DEPENDENT ARGUMENTS - # no backup files if step == steps.ALL or step == steps.PREPARATION: parser.add_argument("--nobak", action="store_true", dest="nobak", default=False, @@ -156,9 +153,7 @@ def getOptions(kinds, step=steps.ALL): help="print relative file names [default: %(default)s]\n" "e.g., '/projects/apache/_cppstats/afile.c.xml' gets 'afile.c.xml'.") - # ADD POSSIBLE PREPARATION/ANALYSIS KINDS AND THEIR COMMAND-LINE ARGUMENTS - if step == steps.ALL: parser.add_argument_group("Possible Kinds of Analyses ".upper(), ", ".join(kinds.keys())) @@ -178,24 +173,16 @@ def getOptions(kinds, step=steps.ALL): for cls in kinds.values(): cls.addCommandLineOptions(parser) - # PARSE OPTIONS - options = parser.parse_args() - # ADD CONSTANTS TO OPTIONS - addConstants(options) - # CHECK CONSTRAINTS ON OPTIONS - checkConstraints(options) - # RETURN - return options @@ -208,6 +195,6 @@ def addConstants(options): def checkConstraints(options): # constraints - if (options.allkinds == True and options.inputfile): - print "Using all kinds of preparation for a single input and output file is weird!" + if options.allkinds is True and options.inputfile: + print("Using all kinds of preparation for a single input and output file is weird!") sys.exit(1) From 9037c93b26343682456a55f65f046fc6c824e49a Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 12:09:19 +0100 Subject: [PATCH 10/72] Fix cppstats/cppstats.py --- cppstats/cppstats.py | 74 +++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index 21b91fe..4c8323b 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -28,50 +28,38 @@ import os import sys -import shutil # for copying files and folders -import errno # for error/exception handling -import threading # for parallelism -import subprocess # for calling other commands -import re # for regular expressions -from abc import ABCMeta, abstractmethod # abstract classes -#import pynotify # for system notifications -from argparse import ArgumentParser, RawTextHelpFormatter # for parameters to this script from collections import OrderedDict # for ordered dictionaries -import tempfile # for temporary files - -# ################################################# -# imports from subfolders +import tempfile # for temporary files # import different kinds of analyses -import cli, preparation, analysis - +import cli +import preparation +import analysis # ################################################# # version number - __version__ = "v0.9.4" -def version() : - return "cppstats " + __version__ + +def version(): + return f"cppstats {__version__}" # ################################################# # collection of analyses # add all kinds of analyses: (name -> (preparation, analysis)) -__kinds = [] -__kinds.append(('general', ('general', 'general'))) -__kinds.append(('generalvalues', ('general', 'generalvalues'))) -__kinds.append(('discipline', ('discipline', 'discipline'))) -__kinds.append(('featurelocations', ('featurelocations', 'featurelocations'))) -__kinds.append(('derivative', ('discipline', 'derivative'))) -__kinds.append(('interaction', ('discipline', 'interaction'))) - +__kinds = [('general', ('general', 'general')), + ('generalvalues', ('general', 'generalvalues')), + ('discipline', ('discipline', 'discipline')), + ('featurelocations', ('featurelocations', 'featurelocations')), + ('derivative', ('discipline', 'derivative')), + ('interaction', ('discipline', 'interaction'))] # exit, if there are no analysis threads available -if (len(__kinds) == 0) : - print "ERROR: No analyses available! Revert your changes or call the maintainer." - print "Exiting now..." +if len(__kinds) == 0: + print("ERROR: No analyses available! Revert your changes or call the maintainer.") + print("Exiting now...") sys.exit(1) __kinds = OrderedDict(__kinds) @@ -82,8 +70,7 @@ def version() : def applyFile(kind, infile, outfile, options): - - tmpfile = tempfile.mkstemp(suffix=".xml")[1] # temporary srcML file + tmpfile = tempfile.mkstemp(suffix=".xml")[1] # temporary srcML file # preparation options.infile = infile @@ -98,6 +85,7 @@ def applyFile(kind, infile, outfile, options): # delete temp file os.remove(tmpfile) + def applyFolders(option_kind, inputlist, options): kind = __kinds.get(option_kind) preparationKind = kind[0] @@ -106,6 +94,7 @@ def applyFolders(option_kind, inputlist, options): preparation.applyFolders(preparationKind, inputlist, options) analysis.applyFolders(analysisKind, inputlist, options) + def applyFoldersAll(inputlist, options): for kind in __kinds.keys(): applyFolders(kind, inputlist, options) @@ -115,41 +104,42 @@ def main(): # ################################################# # options parsing - options = cli.getOptions(__kinds, step = cli.steps.ALL) + options = cli.getOptions(__kinds, step=cli.steps.ALL) # ################################################# # main - if (options.inputfile): + if options.inputfile: # split --file argument - options.infile = os.path.normpath(os.path.abspath(options.inputfile[0])) # IN - options.outfile = os.path.normpath(os.path.abspath(options.inputfile[1])) # OUT + options.infile = os.path.normpath(os.path.abspath(options.inputfile[0])) # IN + options.outfile = os.path.normpath(os.path.abspath(options.inputfile[1])) # OUT # check if inputfile exists - if (not os.path.isfile(options.infile)): - print "ERROR: input file '{}' cannot be found!".format(options.infile) + if not os.path.isfile(options.infile): + print(f"ERROR: input file '{options.infile}' cannot be found!") sys.exit(1) applyFile(options.kind, options.infile, options.outfile, options) - elif (options.inputlist): + elif options.inputlist: # handle --list argument - options.inputlist = os.path.normpath(os.path.abspath(options.inputlist)) # LIST + options.inputlist = os.path.normpath(os.path.abspath(options.inputlist)) # LIST # check if list file exists - if (not os.path.isfile(options.inputlist)): - print "ERROR: input file '{}' cannot be found!".format(options.inputlist) + if not os.path.isfile(options.inputlist): + print(f"ERROR: input file '{options.inputlist}' cannot be found!") sys.exit(1) - if (options.allkinds): + if options.allkinds: applyFoldersAll(options.inputlist, options) else: applyFolders(options.kind, options.inputlist, options) else: - print "This should not happen! No input file or list of projects given!" + print("This should not happen! No input file or list of projects given!") sys.exit(1) + if __name__ == '__main__': main() From 1744ccb58c835cba2407839d8bbd9aee4e52fc35 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 12:15:16 +0100 Subject: [PATCH 11/72] Fix cppstats/preparation.py --- cppstats/preparation.py | 66 +++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 35 deletions(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 101d97b..ab7f667 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -35,12 +35,12 @@ from abc import ABCMeta, abstractmethod # abstract classes from collections import OrderedDict - # ################################################# # paths import preparations + def getPreparationScript(filename): return os.path.join(os.path.dirname(preparations.__file__), filename) @@ -111,7 +111,7 @@ def replaceMultiplePatterns(replacements, infile, outfile): with open(infile, "rb") as source: with open(outfile, "w") as target: data = source.read() - for pattern, replacement in replacements.iteritems(): + for pattern, replacement in replacements.items(): data = re.sub(pattern, replacement, data, flags=re.MULTILINE) target.write(data) @@ -121,7 +121,7 @@ def stripEmptyLinesFromFile(infile, outfile): with open(outfile, "w") as target: for line in source: if line.strip(): - target.write(line) + target.write(line) # TODO Read bytes and write strings?? def silentlyRemoveFile(filename): @@ -147,7 +147,7 @@ def srcml2src(srcml, src): # abstract preparation thread class AbstractPreparationThread(object): - '''This class prepares a single folder according to the given kind of preparations in an independent thread.''' + """This class prepares a single folder according to the given kind of preparations in an independent thread.""" __metaclass__ = ABCMeta sourcefolder = "source" @@ -155,7 +155,7 @@ def __init__(self, options, inputfolder=None, inputfile=None): self.options = options self.notrunnable = False - if (inputfolder): + if inputfolder: self.file = None self.folder = inputfolder self.source = os.path.join(self.folder, self.sourcefolder) @@ -165,7 +165,7 @@ def __init__(self, options, inputfolder=None, inputfile=None): # get full path of subfolder "_cppstats" self.subfolder = os.path.join(self.folder, self.getSubfolder()) - elif (inputfile): + elif inputfile: self.file = inputfile self.outfile = self.options.outfile self.folder = os.path.dirname(self.file) @@ -175,36 +175,33 @@ def __init__(self, options, inputfolder=None, inputfile=None): # get full path of temp folder for import tempfile self.subfolder = tempfile.mkdtemp(suffix=self.getSubfolder()) - - else: self.notrunnable = True def startup(self): # LOGGING - notify("starting '" + self.getPreparationName() + "' preparations:\n " + self.project) - print "# starting '" + self.getPreparationName() + "' preparations: " + self.project + notify(f"starting '{self.getPreparationName()}' preparations:\n {self.project}") + print(f"# starting '{self.getPreparationName()}' preparations: {self.project}") def teardown(self): # delete temp folder for file-based preparation - if (self.file): + if self.file: shutil.rmtree(self.subfolder) # LOGGING - notify("finished '" + self.getPreparationName() + "' preparations:\n " + self.project) - print "# finished '" + self.getPreparationName() + "' preparations: " + self.project + notify(f"finished '{self.getPreparationName()}' preparations:\n{self.project}") + print(f"# finished '{self.getPreparationName()}' preparations: {self.project}") def run(self): - if (self.notrunnable): - print "ERROR: No single file or input list of projects given!" + if self.notrunnable: + print("ERROR: No single file or input list of projects given!") return self.startup() - if (self.file): - + if self.file: self.currentFile = os.path.join(self.subfolder, self.project) shutil.copyfile(self.file, self.currentFile) @@ -227,7 +224,6 @@ def run(self): self.teardown() def copyToSubfolder(self): - # TODO debug # echo '### preparing sources ...' # echo '### copying all-files to one folder ...' @@ -240,8 +236,8 @@ def copyToSubfolder(self): shutil.copytree(self.source, self.subfolder, ignore=filterForFiles) def backupCurrentFile(self): - '''# backup file''' - if (not self.options.nobak): + """# backup file""" + if not self.options.nobak: bak = self.currentFile + ".bak" + str(self.backupCounter) shutil.copyfile(self.currentFile, bak) self.backupCounter += 1 @@ -336,7 +332,7 @@ def rewriteIfdefsAndIfndefs(self): def removeIncludeGuards(self): # include guards only exist in H files, otherwise return _, extension = os.path.splitext(self.currentFile) - if (extension not in _filepattern_h): + if extension not in _filepattern_h: return tmp = self.currentFile + "tmp.txt" @@ -403,7 +399,7 @@ def prepareFile(self): # rewrite #if(n)def ... to #if (!)defined(...) self.rewriteIfdefsAndIfndefs() - # removes include guards from H files + # removes include guards against H files self.removeIncludeGuards() # delete empty lines @@ -434,7 +430,7 @@ def prepareFile(self): # rewrite #if(n)def ... to #if (!)defined(...) self.rewriteIfdefsAndIfndefs() - # removes include guards from H files + # removes include guards against H files self.removeIncludeGuards() # removes other preprocessor than #ifdefs @@ -506,9 +502,9 @@ def prepareFile(self): __preparationkinds.append(entry) # exit, if there are no preparation threads available -if (len(__preparationkinds) == 0): - print "ERROR: No preparation tasks found! Revert your changes or call the maintainer." - print "Exiting now..." +if len(__preparationkinds) == 0: + print("ERROR: No preparation tasks found! Revert your changes or call the maintainer.") + print("Exiting now...") sys.exit(1) __preparationkinds = OrderedDict(__preparationkinds) @@ -531,7 +527,7 @@ def applyFile(kind, inputfile, options): def getFoldersFromInputListFile(inputlist): - ''' This method reads the given inputfile line-wise and returns the read lines without line breaks.''' + """ This method reads the given inputfile line-wise and returns the read lines without line breaks.""" file = open(inputlist, 'r') # open input file folders = file.read().splitlines() # read lines from file without line breaks @@ -579,35 +575,35 @@ def main(): # ################################################# # main - if (options.inputfile): + if options.inputfile: # split --file argument options.infile = os.path.normpath(os.path.abspath(options.inputfile[0])) # IN options.outfile = os.path.normpath(os.path.abspath(options.inputfile[1])) # OUT # check if inputfile exists - if (not os.path.isfile(options.infile)): - print "ERROR: input file '{}' cannot be found!".format(options.infile) + if not os.path.isfile(options.infile): + print(f"ERROR: input file '{options.infile}' cannot be found!") sys.exit(1) applyFile(options.kind, options.infile, options) - elif (options.inputlist): + elif options.inputlist: # handle --list argument options.inputlist = os.path.normpath(os.path.abspath(options.inputlist)) # LIST # check if list file exists - if (not os.path.isfile(options.inputlist)): - print "ERROR: input file '{}' cannot be found!".format(options.inputlist) + if not os.path.isfile(options.inputlist): + print(f"ERROR: input file '{options.inputlist}' cannot be found!") sys.exit(1) - if (options.allkinds): + if options.allkinds: applyFoldersAll(options.inputlist, options) else: applyFolders(options.kind, options.inputlist, options) else: - print "This should not happen! No input file or list of projects given!" + print("This should not happen! No input file or list of projects given!") sys.exit(1) From 217a861c918bc994d4237075b541114729ef7715 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 28 Feb 2023 12:19:08 +0100 Subject: [PATCH 12/72] Fix lib/cpplib.py --- lib/cpplib.py | 89 +++++++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 45 deletions(-) diff --git a/lib/cpplib.py b/lib/cpplib.py index e351675..2173f8a 100644 --- a/lib/cpplib.py +++ b/lib/cpplib.py @@ -26,8 +26,9 @@ # pyparsing module import pyparsing as pypa -pypa.ParserElement.enablePackrat() # speed up parsing -sys.setrecursionlimit(2000) # handle larger expressions + +pypa.ParserElement.enablePackrat() # speed up parsing +sys.setrecursionlimit(2000) # handle larger expressions # possible operands: # - hexadecimal number @@ -36,31 +37,27 @@ # - macro function, which is basically expanded via #define # to an expression __string = \ - pypa.Literal('\'').suppress() + \ - pypa.Word(pypa.alphanums+'_\\') + \ - pypa.Literal('\'').suppress() + pypa.Literal('\'').suppress() + \ + pypa.Word(pypa.alphanums + '_\\') + \ + pypa.Literal('\'').suppress() __hexadec = \ - pypa.Literal('0x').suppress() + \ - pypa.Word(pypa.hexnums).\ - setParseAction(lambda t: str(int(t[0], 16))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + pypa.Literal('0x').suppress() + \ + pypa.Word(pypa.hexnums).setParseAction(lambda t: str(int(t[0], 16))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('L'))) __integer = \ - pypa.Optional('~') + \ - pypa.Word(pypa.nums+'-').setParseAction(lambda t: str(int(t[0]))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('U'))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + \ - pypa.Optional(pypa.Suppress(pypa.Literal('L'))) - -__identifier = \ - pypa.Word(pypa.alphanums+'_'+'-') -__arg = pypa.Word(pypa.alphanums+'_') -__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + \ - __arg) + pypa.Optional('~') + \ + pypa.Word(pypa.nums + '-').setParseAction(lambda t: str(int(t[0]))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('U'))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + \ + pypa.Optional(pypa.Suppress(pypa.Literal('L'))) + +__identifier = pypa.Word(pypa.alphanums + '_' + '-') +__arg = pypa.Word(pypa.alphanums + '_') +__args = __arg + pypa.ZeroOrMore(pypa.Literal(',').suppress() + __arg) __fname = pypa.Word(pypa.alphas, pypa.alphanums + '_') -__function = pypa.Group(__fname + pypa.Literal('(').suppress() + \ - __args + pypa.Literal(')').suppress()) +__function = pypa.Group(__fname + pypa.Literal('(').suppress() + __args + pypa.Literal(')').suppress()) def _parseIfDefExpression(ifdefexp): @@ -75,9 +72,11 @@ def _rewriteOne(param): representation for csp.""" op, ma = param[0] mal.append(ma) - if op == '!': ret = op + '(' + ma + ')' - if op == 'defined': ret = ma - return ret + if op == '!': + ret = op + '(' + ma + ')' + if op == 'defined': + ret = ma + return ret def _rewriteTwo(param): """This function returns each two parameter function @@ -87,55 +86,54 @@ def _rewriteTwo(param): ret = '(' + ret.join(map(str, param[0][0::2])) + ')' return ret - operand = __string | __hexadec | __integer | \ - __function | __identifier - operators = pypa.oneOf('&& ||') # extend with furhter operators + operand = __string | __hexadec | __integer | __function | __identifier + operators = pypa.oneOf('&& ||') # extend with furhter operators expr = pypa.operatorPrecedence(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), - ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), + ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (operators, 2, pypa.opAssoc.LEFT, _rewriteTwo), ]) + pypa.StringEnd() try: rsig = expr.parseString(ifdefexp)[0] except pypa.ParseException as e: - print('ERROR (parse): cannot parse sig (%s) -- (%s)' % - (ifdefexp, e.col)) + print(f'ERROR (parse): cannot parse sig ({ifdefexp}) -- ({e.col})') return ifdefexp except RuntimeError: - print('ERROR (time): cannot parse sig (%s)' % (ifdefexp)) + print(f'ERROR (time): cannot parse sig ({ifdefexp})') return ifdefexp - return (mal, ''.join(rsig)) + return mal, ''.join(rsig) __ifdefexplist = [] + + def _collectIfdefExpressions(fname): - ''' + """ This method filters all ifdef expressions out of a file and returns them as a list. - ''' + """ def _extractIfdefExpression(tokens): global __ifdefexplist __ifdefexplist += tokens - __macro = pypa.Literal('#') \ - + pypa.oneOf("if ifdef ifndef elif") \ - + pypa.OneOrMore(pypa.Word(pypa.alphanums+"&|><=^")) \ - .setParseAction(_extractIfdefExpression) \ - + pypa.StringEnd() + __macro = pypa.Literal('#') + pypa.oneOf("if ifdef ifndef elif") \ + + pypa.OneOrMore(pypa.Word(pypa.alphanums + "&|><=^")).setParseAction(_extractIfdefExpression) \ + + pypa.StringEnd() with open(fname, 'r') as fd: - for line in fd.xreadlines(): + for line in fd.readlines(): try: print(__macro.parseString(line)) except pypa.ParseException: pass return __ifdefexplist + def _filterAnnotatedIfdefs(fnamein, fnameout): - ''' + """ This method removes all preprocessor annotated lines from the input. - ''' + """ inifdef = 0 with open(fnameout, 'w') as fdout: @@ -152,12 +150,13 @@ def _filterAnnotatedIfdefs(fnamein, fnameout): pass elif directive == '#endif': inifdef -= 1 - elif directive in ['#line', '#error', '#pragma', '#define', '#undef', '#include', '#ident', '#warning', '#include_next']: + elif directive in ['#line', '#error', '#pragma', '#define', '#undef', '#include', '#ident', + '#warning', '#include_next']: if inifdef: fdout.write('\n') continue else: - print("ERROR: directive (%s) not processed!" % parseddirective) + print(f"ERROR: directive ({parseddirective}) not processed!") fdout.write(line) # found regular C code else: From 7dd39da0a575e5c4dda30ad1fe17336576f947b3 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 8 Mar 2023 08:57:14 +0100 Subject: [PATCH 13/72] Fix preparations/deleteIncludeGuards.py --- preparations/deleteIncludeGuards.py | 32 ++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/preparations/deleteIncludeGuards.py b/preparations/deleteIncludeGuards.py index d9191b3..4526705 100755 --- a/preparations/deleteIncludeGuards.py +++ b/preparations/deleteIncludeGuards.py @@ -41,7 +41,9 @@ # line x+2: ... # line x+y: #endif -import os, re, sys +import os +import re +import sys __debug = False @@ -91,22 +93,22 @@ def _findCorrespondingItems(sourcecode): # end of code reached and nothing found so far if currentitem == len(sourcecode): # no include guard found - return (-1, -1) + return -1, -1 # processing ifdef and define regres = rg.match(sifdef) - if (regres): + if regres: guardname = regres.groups()[0] else: - return (-1, -1) + return -1, -1 define = taillist[1] regres = rd.match(define.strip()) - if (regres): + if regres: if guardname != regres.groups()[0]: - return (-1, -1) + return -1, -1 else: - return (-1, -1) + return -1, -1 # process taillist for else and endif ifcount = 1 @@ -114,28 +116,26 @@ def _findCorrespondingItems(sourcecode): for item in taillist[2:]: currentitem += 1 if item.startswith('#else') and ifcount == 1: - return (-1, -1) # we do not support alternative include guards + return -1, -1 # we do not support alternative include guards if item.startswith('#elif') and ifcount == 1: - return (-1, -1) # we do not support alternative include guards + return -1, -1 # we do not support alternative include guards if item.startswith('#if'): ifcount += 1 continue if item.startswith('#endif'): ifcount -= 1 if ifcount == 0: - return (ifdefpos, currentitem + ifdefpos) + return ifdefpos, currentitem + ifdefpos continue - return (-1, -1) + return -1, -1 (ifdef, endif) = _findCorrespondingItems(list(sourcecode)) - if (ifdef == -1 or endif == -1): + if ifdef == -1 or endif == -1: pass else: # concat source code again and replace include guard with empty lines - sourcecode = sourcecode[:max(0, ifdef)] + \ - ["", ""] + \ - sourcecode[ifdef + 2:endif] + \ - [""] + \ + sourcecode = sourcecode[:max(0, ifdef)] + ["", ""] + \ + sourcecode[ifdef + 2:endif] + [""] + \ sourcecode[endif + 1:] for item in sourcecode: From 696c2f9e760736310f96e431c6aec81034d22027 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 8 Mar 2023 08:58:41 +0100 Subject: [PATCH 14/72] Fix preparations/rewriteIfdefs.py --- preparations/rewriteIfdefs.py | 55 +++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 25 deletions(-) diff --git a/preparations/rewriteIfdefs.py b/preparations/rewriteIfdefs.py index c62e900..6ddf870 100755 --- a/preparations/rewriteIfdefs.py +++ b/preparations/rewriteIfdefs.py @@ -22,38 +22,43 @@ # Jörg Liebig # Claus Hunsen -import sys, os +import sys + class WrongIfdefError(Exception): - def __init__(self): - pass - def __str__(self): - return ("Didn't find \"ifdef\" or \"ifndef\" as macro") + def __init__(self): + pass + + def __str__(self): + return "Didn't find \"ifdef\" or \"ifndef\" as macro" + -def rewriteFile(fname, out = sys.stdout): - fd = open(fname, 'r') +def rewriteFile(fname, out=sys.stdout): + fd = open(fname, 'r') - for line in fd: - if line.startswith('#ifdef') or line.startswith('#ifndef'): - ifdef, identifier = line.split(None, 1) # FIXME if there is a comment after the constant, it is incorporated into the brackets! this may lead to errors. - identifier = identifier.strip() + for line in fd: + if line.startswith('#ifdef') or line.startswith('#ifndef'): + # FIXME if there is a comment after the constant, it is incorporated into the brackets! this may lead to + # errors. + ifdef, identifier = line.split(None, 1) + identifier = identifier.strip() - if ifdef == '#ifdef': - out.write('#if defined(' + identifier + ')' + '\n') - continue - if ifdef == '#ifndef': - out.write('#if !defined(' + identifier + ')' + '\n') - continue - raise WrongIfdefError() - else: - out.write(line) + if ifdef == '#ifdef': + out.write('#if defined(' + identifier + ')' + '\n') + continue + if ifdef == '#ifndef': + out.write('#if !defined(' + identifier + ')' + '\n') + continue + raise WrongIfdefError() + else: + out.write(line) - fd.close() + fd.close() ################################################## if __name__ == '__main__': - if (len(sys.argv) != 2): - print("usage: " + sys.argv[0] + " ") - else: - rewriteFile(sys.argv[1]) + if len(sys.argv) != 2: + print("usage: " + sys.argv[0] + " ") + else: + rewriteFile(sys.argv[1]) From e66c4f2c393c7ae0c68e07d10ba3972c65150886 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 8 Mar 2023 08:59:23 +0100 Subject: [PATCH 15/72] Fix preparations/rewriteMultilineMacros.py --- preparations/rewriteMultilineMacros.py | 98 +++++++++++++------------- 1 file changed, 50 insertions(+), 48 deletions(-) diff --git a/preparations/rewriteMultilineMacros.py b/preparations/rewriteMultilineMacros.py index b112b54..22b3cce 100755 --- a/preparations/rewriteMultilineMacros.py +++ b/preparations/rewriteMultilineMacros.py @@ -32,64 +32,66 @@ # # effects all kind of macros #defines, conditionals, ... -import os, sys +import os +import sys + # translates macros (s.o. or usage) def translate(infile, outfile): - fdin = open(infile) - fdout = open(outfile, 'w') - curline = '' - numlines = 0 - - for line in fdin: - sline = line.strip() - - # macro found - if len(curline): - # multi-line macro - if sline.endswith('\\'): - curline += sline[:-1] - numlines += 1 - else: - curline += sline - #TODO fix line endings - fdout.write(curline+'\n') - fdout.write('\n' * numlines) - curline = '' - numlines = 0 - - continue - - # found a new macro - if (sline.startswith('#') and sline.endswith('\\')): - curline += sline[:-1] - numlines += 1 - continue - - # normal line - fdout.write(line) - - # closeup - fdin.close() - fdout.close() + fdin = open(infile) + fdout = open(outfile, 'w') + curline = '' + numlines = 0 + + for line in fdin: + sline = line.strip() + + # macro found + if len(curline): + # multi-line macro + if sline.endswith('\\'): + curline += sline[:-1] + numlines += 1 + else: + curline += sline + # TODO fix line endings + fdout.write(curline + '\n') + fdout.write('\n' * numlines) + curline = '' + numlines = 0 + + continue + + # found a new macro + if sline.startswith('#') and sline.endswith('\\'): + curline += sline[:-1] + numlines += 1 + continue + + # normal line + fdout.write(line) + + # closeup + fdin.close() + fdout.close() # usage def usage(): - print('usage: ' + sys.argv[0] + ' ') - print('') - print('Translates multiple macros in the source-code of the infile') - print('to a oneliner-macro in the outfile.') + print('usage: ' + sys.argv[0] + ' ') + print('') + print('Translates multiple macros in the source-code of the infile') + print('to a oneliner-macro in the outfile.') + ################################################## if __name__ == '__main__': - if (len(sys.argv) < 3): - usage() - sys.exit(-1) - - infile = os.path.abspath(sys.argv[1]) - outfile = os.path.abspath(sys.argv[2]) + if len(sys.argv) < 3: + usage() + sys.exit(-1) - translate(infile, outfile) + infile = os.path.abspath(sys.argv[1]) + outfile = os.path.abspath(sys.argv[2]) + translate(infile, outfile) From b9d8670299e3c4b4bfb2d58b59110548ca559ced Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 8 Mar 2023 09:02:24 +0100 Subject: [PATCH 16/72] Fix scripts/ascope.py --- scripts/ascope.py | 372 +++++++++++++++++++++++----------------------- 1 file changed, 186 insertions(+), 186 deletions(-) diff --git a/scripts/ascope.py b/scripts/ascope.py index e0cae6d..09cf8ef 100755 --- a/scripts/ascope.py +++ b/scripts/ascope.py @@ -32,197 +32,197 @@ # external libs # python-lxml module try: - from lxml import etree + from lxml import etree except ImportError: - print("python-lxml module not found! (python-lxml)") - print("see http://codespeak.net/lxml/") - print("programm terminating ...!") - sys.exit(-1) - - -def returnFileNames(folder, extfilt = ['.xml']): - '''This function returns all files of the input folder - and its subfolders.''' - filesfound = list() - - if os.path.isdir(folder): - wqueue = [os.path.abspath(folder)] - - while wqueue: - currentfolder = wqueue[0] - wqueue = wqueue[1:] - foldercontent = os.listdir(currentfolder) - tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) - tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) - tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) - filesfound += tmpfiles - tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) - tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) - wqueue += tmpfolders - - return filesfound + print("python-lxml module not found! (python-lxml)") + print("see http://codespeak.net/lxml/") + print("programm terminating ...!") + sys.exit(-1) + + +def returnFileNames(folder, extfilt=['.xml']): + """This function returns all files of the input folder + and its subfolders.""" + filesfound = list() + + if os.path.isdir(folder): + wqueue = [os.path.abspath(folder)] + + while wqueue: + currentfolder = wqueue[0] + wqueue = wqueue[1:] + foldercontent = os.listdir(currentfolder) + tmpfiles = filter(lambda n: os.path.isfile( + os.path.join(currentfolder, n)), foldercontent) + tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, + tmpfiles) + tmpfiles = map(lambda n: os.path.join(currentfolder, n), + tmpfiles) + filesfound += tmpfiles + tmpfolders = filter(lambda n: os.path.isdir( + os.path.join(currentfolder, n)), foldercontent) + tmpfolders = map(lambda n: os.path.join(currentfolder, n), + tmpfolders) + wqueue += tmpfolders + + return filesfound class Ascope: - ################################################## - # constants: - __cppnscpp = 'http://www.srcML.org/srcML/cpp' - __cppnsdef = 'http://www.srcML.org/srcML/src' - __cpprens = re.compile('{(.+)}(.+)') - __conditionals = ['if', 'ifdef', 'ifndef', 'else', 'elif', 'endif'] - __conditions = ['if', 'ifdef', 'ifndef'] - __screensize = 50 - __depthannotation = 60 - ################################################## - - def __init__(self): - oparser = OptionParser() - oparser.add_option('-d', '--dir', dest='dir', - help='input directory (mandatory)') - (self.opts, self.args) = oparser.parse_args() - - if not self.opts.dir: - oparser.print_help() - sys.exit(-1) - - self.loc=0 - self.checkFiles() - - def __getIfdefAnnotations__(self, root): - '''This method returns all nodes of the xml which are ifdef - annotations in the source code.''' - treeifdefs = list() - - for _, elem in etree.iterwalk(root): - ns, tag = Ascope.__cpprens.match(elem.tag).\ - groups() - - if ns == Ascope.__cppnscpp \ - and tag in Ascope.__conditionals: - treeifdefs.append(elem) - - return treeifdefs - - def __createListFromTreeifdefs__(self, treeifdefs): - '''This method returns a list representation for the input treeifdefs - (xml-objects). Corresponding #ifdef elements are in one sublist.''' - try: - if not treeifdefs: return [] - - listifdefs = list() - workerlist = list() - for nifdef in treeifdefs: - tag = nifdef.tag.split('}')[1] - if tag in ['if', 'ifdef', 'ifndef']: - workerlist.append(list()) - workerlist[-1].append(nifdef) - elif tag in ['elif', 'else']: - workerlist[-1].append(nifdef) - elif tag in ['endif']: - workerlist[-1].append(nifdef) - listifdefs.append(workerlist[-1]) - workerlist = workerlist[:-1] - else: - print('ERROR: tag (%s) unknown!' % tag) - - return listifdefs - except IndexError: - return [] - - def __getParentTag__(self, tag): - parent = tag.getparent() - return parent.tag.split('}')[1] - - - def __checkDiscipline__(self, treeifdefs, loc, stats, statsU): - listundisciplined = self.__createListFromTreeifdefs__(treeifdefs) - # print('INFO: %s annotations to check' % len(listundisciplined)) - - allannotations=[] - for ifdef in listundisciplined: - for i in range(len(ifdef)-1): - allannotations.append([ifdef[i].sourceline,ifdef[i+1].sourceline,self.__findFeatures__(ifdef,i)]); - - for screen in range(0, max(1,min(65000,loc-Ascope.__screensize/2)), Ascope.__screensize/2): - screenend=min(loc, screen+Ascope.__screensize) - annotationsOnScreen=set() - annotationsOnScreenCount=0 - for annotation in allannotations: - if annotation[0]<=screenend: - if annotation[1]>screen: - annotationsOnScreen.add(annotation[2]) - annotationsOnScreenCount=annotationsOnScreenCount+1 - try: - stats[annotationsOnScreenCount]=stats[annotationsOnScreenCount]+1 - statsU[len(annotationsOnScreen)]=statsU[len(annotationsOnScreen)]+1 - except IndexError: - print(annotationsOnScreenCount) - sys.exit(-1) - - # print(stats) - # print(statsU) - - def __findFeatures__(self, ifdef, idx): - result="" - if ifdef[idx].tag.split('}')[1]=='else': - idx=0 - result="!" - if ifdef[idx].tag.split('}')[1]=='ifndef': - if (result=="!"): - result="" - else: - result="!" - - context = etree.iterwalk(ifdef[idx]) - for action, elem in context: - if action=="end": - if elem.tag.split('}')[1]=="name": - result=result+elem.text - # print result; - return result - - def checkFile(self, file, stats, statsU): - # print('INFO: processing (%s)' % file) - - try: - tree = etree.parse(file) - - f = open(file, 'r') - except etree.XMLSyntaxError: - print('ERROR: file (%s) is not valid. Skipping it.' % file) - return - - #get LOC - thisloc=len(f.readlines())-2 - if (thisloc > 65000): - print('INFO: file (%s) not fully processed!' % file) - - # get root of the xml and iterate over it - root = tree.getroot() - treeifdefs = self.__getIfdefAnnotations__(root) - self.__checkDiscipline__(treeifdefs, thisloc, stats, statsU) - - - def checkFiles(self): - xmlfiles = returnFileNames(self.opts.dir, ['.xml']) - stats=[0]*Ascope.__depthannotation - statsU=[0]*Ascope.__depthannotation - for xmlfile in xmlfiles: - self.checkFile(xmlfile, stats, statsU) - f = open("count.csv","a") - f.write(self.opts.dir+";"+str(Ascope.__screensize)+";") - for i in stats: - f.write(str(i)+";") - for i in statsU: - f.write(str(i)+";") - f.write("\n") + ################################################## + # constants: + __cppnscpp = 'http://www.srcML.org/srcML/cpp' + __cppnsdef = 'http://www.srcML.org/srcML/src' + __cpprens = re.compile('{(.+)}(.+)') + __conditionals = ['if', 'ifdef', 'ifndef', 'else', 'elif', 'endif'] + __conditions = ['if', 'ifdef', 'ifndef'] + __screensize = 50 + __depthannotation = 60 + + ################################################## + + def __init__(self): + oparser = OptionParser() + oparser.add_option('-d', '--dir', dest='dir', + help='input directory (mandatory)') + (self.opts, self.args) = oparser.parse_args() + + if not self.opts.dir: + oparser.print_help() + sys.exit(-1) + + self.loc = 0 + self.checkFiles() + + def __getIfdefAnnotations__(self, root): + """This method returns all nodes of the xml which are ifdef + annotations in the source code.""" + treeifdefs = list() + + for _, elem in etree.iterwalk(root): + ns, tag = Ascope.__cpprens.match(elem.tag). \ + groups() + + if ns == Ascope.__cppnscpp \ + and tag in Ascope.__conditionals: + treeifdefs.append(elem) + + return treeifdefs + + def __createListFromTreeifdefs__(self, treeifdefs): + """This method returns a list representation for the input treeifdefs + (xml-objects). Corresponding #ifdef elements are in one sublist.""" + try: + if not treeifdefs: + return [] + + listifdefs = list() + workerlist = list() + for nifdef in treeifdefs: + tag = nifdef.tag.split('}')[1] + if tag in ['if', 'ifdef', 'ifndef']: + workerlist.append(list()) + workerlist[-1].append(nifdef) + elif tag in ['elif', 'else']: + workerlist[-1].append(nifdef) + elif tag in ['endif']: + workerlist[-1].append(nifdef) + listifdefs.append(workerlist[-1]) + workerlist = workerlist[:-1] + else: + print('ERROR: tag (%s) unknown!' % tag) + + return listifdefs + except IndexError: + return [] + + def __getParentTag__(self, tag): + parent = tag.getparent() + return parent.tag.split('}')[1] + + def __checkDiscipline__(self, treeifdefs, loc, stats, statsU): + listundisciplined = self.__createListFromTreeifdefs__(treeifdefs) + # print('INFO: %s annotations to check' % len(listundisciplined)) + + allannotations = [] + for ifdef in listundisciplined: + for i in range(len(ifdef) - 1): + allannotations.append([ifdef[i].sourceline, ifdef[i + 1].sourceline, self.__findFeatures__(ifdef, i)]) + + for screen in range(0, max(1, min(65000, loc - Ascope.__screensize / 2)), Ascope.__screensize / 2): + screenend = min(loc, screen + Ascope.__screensize) + annotationsOnScreen = set() + annotationsOnScreenCount = 0 + for annotation in allannotations: + if annotation[0] <= screenend: + if annotation[1] > screen: + annotationsOnScreen.add(annotation[2]) + annotationsOnScreenCount = annotationsOnScreenCount + 1 + try: + stats[annotationsOnScreenCount] = stats[annotationsOnScreenCount] + 1 + statsU[len(annotationsOnScreen)] = statsU[len(annotationsOnScreen)] + 1 + except IndexError: + print(annotationsOnScreenCount) + sys.exit(-1) + + # print(stats) + # print(statsU) + + def __findFeatures__(self, ifdef, idx): + result = "" + if ifdef[idx].tag.split('}')[1] == 'else': + idx = 0 + result = "!" + if ifdef[idx].tag.split('}')[1] == 'ifndef': + if result == "!": + result = "" + else: + result = "!" + + context = etree.iterwalk(ifdef[idx]) + for action, elem in context: + if action == "end": + if elem.tag.split('}')[1] == "name": + result = result + elem.text + # print result; + return result + + def checkFile(self, file, stats, statsU): + # print('INFO: processing (%s)' % file) + + try: + tree = etree.parse(file) + + f = open(file, 'r') + except etree.XMLSyntaxError: + print('ERROR: file (%s) is not valid. Skipping it.' % file) + return + + # get LOC + thisloc = len(f.readlines()) - 2 + if thisloc > 65000: + print('INFO: file (%s) not fully processed!' % file) + + # get root of the xml and iterate over it + root = tree.getroot() + treeifdefs = self.__getIfdefAnnotations__(root) + self.__checkDiscipline__(treeifdefs, thisloc, stats, statsU) + + def checkFiles(self): + xmlfiles = returnFileNames(self.opts.dir, ['.xml']) + stats = [0] * Ascope.__depthannotation + statsU = [0] * Ascope.__depthannotation + for xmlfile in xmlfiles: + self.checkFile(xmlfile, stats, statsU) + f = open("count.csv", "a") + f.write(self.opts.dir + ";" + str(Ascope.__screensize) + ";") + for i in stats: + f.write(str(i) + ";") + for i in statsU: + f.write(str(i) + ";") + f.write("\n") ################################################## if __name__ == '__main__': - Ascope() + Ascope() From 3adaa720304ee99a2282150394db9b21a92f1d89 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 8 Mar 2023 09:04:23 +0100 Subject: [PATCH 17/72] Fix scripts/ifdefendifratio.py --- scripts/ifdefendifratio.py | 93 +++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 46 deletions(-) diff --git a/scripts/ifdefendifratio.py b/scripts/ifdefendifratio.py index 1fa2290..ad4509f 100755 --- a/scripts/ifdefendifratio.py +++ b/scripts/ifdefendifratio.py @@ -29,16 +29,17 @@ # modules from the std-library -import os, re, sys +import os +import re +import sys try: - from lxml import etree + from lxml import etree except ImportError: - print("python-lxml module not found! (python-lxml)") - print("see http://codespeak.net/lxml/") - print("programm terminating ...!") - sys.exit(-1) - + print("python-lxml module not found! (python-lxml)") + print("see http://codespeak.net/lxml/") + print("programm terminating ...!") + sys.exit(-1) ################################################## # constants: @@ -49,67 +50,67 @@ __conditionals = ['if', 'ifdef', 'ifndef'] __conditionals_endif = ['endif'] + + ################################################## def _getIfdefEndifRatio(root): - """This function determines all conditionals and their corresponding + """This function determines all conditionals and their corresponding endifs and returns a counter for each of them.""" - ifdef = 0 - endif = 0 + ifdef = 0 + endif = 0 - # get all nodes - allnodes = [node for node in root.iterdescendants()] + # get all nodes + allnodes = [node for node in root.iterdescendants()] - for node in allnodes: - ns, tag = __cpprens.match(node.tag).groups() + for node in allnodes: + ns, tag = __cpprens.match(node.tag).groups() - if ((tag in __conditionals) \ - and (ns == __cppnscpp)): - ifdef += 1 - if ((tag in __conditionals_endif) \ - and (ns == __cppnscpp)): - endif += 1 + if (tag in __conditionals) and (ns == __cppnscpp): + ifdef += 1 + if (tag in __conditionals_endif) and (ns == __cppnscpp): + endif += 1 - return (ifdef, endif) + return ifdef, endif def apply(folder): - """This function applies the determination function (getIfdefEndifRation) + """This function applies the determination function (getIfdefEndifRation) to each file and prints out the differance in case there is one.""" - folder = os.path.abspath(folder) - files = os.listdir(folder) - files = filter(lambda n: os.path.splitext(n)[1] == ".xml", files) + folder = os.path.abspath(folder) + files = os.listdir(folder) + files = filter(lambda n: os.path.splitext(n)[1] == ".xml", files) - for file in files: + for file in files: - try: - tree = etree.parse(file) - except etree.XMLSyntaxError: - print("ERROR: cannot parse (%s). Skipping this file!." % file) + try: + tree = etree.parse(file) + except etree.XMLSyntaxError: + print("ERROR: cannot parse (%s). Skipping this file!." % file) - root = tree.getroot() - ifdef, endif = _getIfdefEndifRatio(root) + root = tree.getroot() + ifdef, endif = _getIfdefEndifRatio(root) - if (ifdef != endif): - print("INFO: (%30s) ifdef : endif is %5s : %5s" % (file, str(ifdef), str(endif))) + if ifdef != endif: + print("INFO: (%30s) ifdef : endif is %5s : %5s" % (file, str(ifdef), str(endif))) def usage(): - """This function prints usage-informations to stdout.""" - print('usage:') - print(' ' + sys.argv[0] + ' ') + """This function prints usage-informations to stdout.""" + print('usage:') + print(' ' + sys.argv[0] + ' ') ################################################## if __name__ == '__main__': - if (len(sys.argv) < 2): - usage() - sys.exit(-1) + if len(sys.argv) < 2: + usage() + sys.exit(-1) - folder = sys.argv[1] - if (os.path.isdir(folder)): - apply(folder) - else: - usage() - sys.exit(-1) + folder = sys.argv[1] + if os.path.isdir(folder): + apply(folder) + else: + usage() + sys.exit(-1) From 10ad5957ee12d0bf34e61129a83a6a2e978c8293 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 8 Mar 2023 09:04:56 +0100 Subject: [PATCH 18/72] Fix scripts/partial_preprocessor.py --- scripts/partial_preprocessor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/partial_preprocessor.py b/scripts/partial_preprocessor.py index ab43a0e..a6419c0 100755 --- a/scripts/partial_preprocessor.py +++ b/scripts/partial_preprocessor.py @@ -23,7 +23,7 @@ # Claus Hunsen -#FIXME move this script to another folder! +# FIXME move this script to another folder! from optparse import OptionParser import sys @@ -34,9 +34,9 @@ class PartialPreprocessor: def __init__(self): oparser = OptionParser() oparser.add_option('-i', '--inputfile', dest='ifile', - help='input file (mandatory)') + help='input file (mandatory)') oparser.add_option('-o', '--outputfile', dest='ofile', - help='output file (mandatory)') + help='output file (mandatory)') (self.opts, self.args) = oparser.parse_args() if not self.opts.ifile or not self.opts.ofile: From f333c664fbd14e31a9bcb74ad8dd31df2fcc062e Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 8 Mar 2023 09:10:54 +0100 Subject: [PATCH 19/72] Fix scripts/reversecpp.py and add missing dependency to setup.py --- scripts/reversecpp.py | 58 +++++++++++++++++++++++-------------------- setup.py | 1 + 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/scripts/reversecpp.py b/scripts/reversecpp.py index 4757d43..1c98e21 100644 --- a/scripts/reversecpp.py +++ b/scripts/reversecpp.py @@ -21,7 +21,7 @@ # Jörg Liebig -''' +""" this collection of functions is dedicated to discipline undisciplined preprocessor annotations in the source code. the annotations covered are those of conditional compilation (#if, #ifdef, #ifndef, #elif, #else, and @@ -37,7 +37,7 @@ 4. based on the line markers move undisciplined annotations to disciplined ones in each of the variants 5. merge the different variants and create a single source code file -''' +""" import itertools import os @@ -54,15 +54,17 @@ src2srcml = os.path.join(os.path.expanduser('~'), 'bin', 'src2srcml2009') src2srcmloptions = ['--language', 'C'] srcml2src = os.path.join(os.path.expanduser('~'), 'bin', 'srcml2src2009') + + ############################################################ class Util: @classmethod - def returnFileNames(folder, extfilt = ['.xml']): - ''' + def returnFileNames(folder, extfilt=['.xml']): + """ This function returns all files of the input folder and its subfolders. - ''' + """ filesfound = list() if os.path.isdir(folder): @@ -73,16 +75,16 @@ def returnFileNames(folder, extfilt = ['.xml']): wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) + tmpfiles) tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles) filesfound += tmpfiles tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) + os.path.join(currentfolder, n)), foldercontent) tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders) wqueue += tmpfolders return filesfound @@ -115,7 +117,7 @@ def setup(self): sys.exit(-1) def addingLineMarkersToFile(self, infile, outfile): - ''' + """ This method adds line markers (comments) to the source code file (infile) and writes the result to the output file (outfile). Three different markers are added: 1. a comment containing the conditional compilation macro is added before each @@ -123,7 +125,7 @@ def addingLineMarkersToFile(self, infile, outfile): 2. each line gets a comment with the lineid at the end 3. include macros are turned into comments so that that preprocessor omits file inclusion during the preprocessing step - ''' + """ fdin = open(os.path.abspath(infile), 'r') fdout = open(os.path.abspath(outfile), 'w') lineid = 0 @@ -131,10 +133,10 @@ def addingLineMarkersToFile(self, infile, outfile): for line in fdin.xreadlines(): # found #if{ndef|def||} or #e{ndif|lse|lif} if line.startswith('#if') or line.startswith('#e'): - fdout.write('//'+ line + str(lineid)) + fdout.write('//' + line + str(lineid)) lineid += 1 if line.startswith('#include'): - fdout.write('//'+ line + str(lineid)) + fdout.write('//' + line + str(lineid)) lineid += 1 continue fdout.write(line.strip() + '/* lineid=' + str(lineid) + ' */\n') @@ -143,20 +145,22 @@ def addingLineMarkersToFile(self, infile, outfile): print('processed file ' + os.path.abspath(infile)) def createVariants(self, symbols, fname): - ''' + """ Generate for each combination of symbols a variant for the inputfile fname and return the list of generated files. - ''' + """ generatedfiles = [] for configuration in itertools.product(range(2), repeat=len(symbols)): configuration = list(configuration) pairs = zip(configuration, symbols) validpairs = filter(lambda (m, n): m != 0, pairs) - if len(validpairs): validdefines = list(zip(*validpairs)[1]) - else: validdefines = [] + if len(validpairs): + validdefines = list(zip(*validpairs)[1]) + else: + validdefines = [] - validdefines = map(lambda n: '-D'+n, validdefines) + validdefines = map(lambda n: '-D' + n, validdefines) cppinvocation = [cpptool] cppinvocation += validdefines cppinvocation += [fname] @@ -172,30 +176,29 @@ def createVariants(self, symbols, fname): return generatedfiles def createXMLRepresentation(self, fname): - ''' + """ This method creates an xml representation from the input file using the src2srcml tool (http://www.srcML.org/). After the successful generation of the xml representation the method returns the filename of the xml file. - ''' + """ src2srcmlinvocation = [src2srcml] src2srcmlinvocation += src2srcmloptions - src2srcmlinvocation += [fname] # input file - src2srcmlinvocation += [fname+'.xml'] # output file + src2srcmlinvocation += [fname] # input file + src2srcmlinvocation += [fname + '.xml'] # output file print(src2srcmlinvocation) subprocess.call(src2srcmlinvocation) - return fname+'.xml' + return fname + '.xml' def createXMLRepresenations(self, flist): - ''' + """ This method creates an xml representation of each file in the input list (flist) and returns a list of the generated xml files. - ''' + """ generatedlist = [] for file in flist: generatedlist.append(self.createXMLRepresentation(file)) return generatedlist - def apply(self): self.setup() symbols, _ = _parseIfDefExpression('AA && BB') @@ -204,6 +207,7 @@ def apply(self): print(flist) print(_collectIfdefExpressions('/home/joliebig/workspace/reverse_cpp/test/test.c')) + ################################################## if __name__ == '__main__': r = ReverseCPP() diff --git a/setup.py b/setup.py index a94477c..2bf3b85 100644 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ #'pyparsing==2.*', #'enum34', #'lxml>=3.4' + #'cpplib' # TODO This was missing but required ], dependency_links=[ From 3200fad117a2e7cd7313c449845745237b21b43b Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 22 Mar 2023 11:42:56 +0100 Subject: [PATCH 20/72] Fix access to keys without casting to list --- cppstats/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/cli.py b/cppstats/cli.py index 5c5bbe1..a7f5d98 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -111,7 +111,7 @@ def getOptions(kinds, step=steps.ALL): # kinds kindgroup = parser.add_mutually_exclusive_group(required=False) kindgroup.add_argument("--kind", choices=kinds.keys(), dest="kind", - default=kinds.keys()[0], metavar="", + default=list(kinds.keys())[0], metavar="", help="the preparation to be performed [default: %(default)s]") kindgroup.add_argument("-a", "--all", action="store_true", dest="allkinds", default=False, help="perform all available kinds of preparation/analysis [default: %(default)s]") From c628ee9869492a11c421d9261264ffbd5132ce48 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 22 Mar 2023 12:08:39 +0100 Subject: [PATCH 21/72] debug debug debug debug debug debug debug debug debug debug debug debug debug debug debug --- analyses/derivative.py | 6 +++--- analyses/featurelocations.py | 2 +- analyses/general.py | 18 +++++++++--------- analyses/generalvalues.py | 8 ++++---- analyses/interaction.py | 10 +++++----- cppstats/cli.py | 4 ++-- cppstats/preparation.py | 6 +++--- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/analyses/derivative.py b/analyses/derivative.py index e3b2b73..44ca4d1 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -774,14 +774,14 @@ def _getFeatureStats(features): lofmean = 0 lofstd = 0 nof = len(features.keys()) - tmp = [item for (_, item) in features.itervalues()] + tmp = [item for (_, item) in list(features.values())] tmp = _flatten(tmp) floflist = list(map(lambda n: n.count('\n'), tmp)) if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = list(reduce(lambda m, n: m + n, floflist)) + lof = reduce(lambda m, n: m + n, floflist) lofmean = np.mean(floflist) if len(floflist) > 1: @@ -1086,7 +1086,7 @@ def _mergeFeatures(ffeatures): for a, f in annotations2andmore: annotationmap[a] = f - annotations2andmore = list(map(lambda s: set(s), annotationmap.values())) + annotations2andmore = list(map(lambda s: set(s), list(annotationmap.values()))) projectpath = os.path.dirname(folder) fd = open(os.path.join(projectpath, __outputfile), 'w') diff --git a/analyses/featurelocations.py b/analyses/featurelocations.py index 6fa02a0..6e1e542 100644 --- a/analyses/featurelocations.py +++ b/analyses/featurelocations.py @@ -675,7 +675,7 @@ def apply(folder, options): continue # parse features and get all defined configuration constants - for (sig, (depth, code)) in features.iteritems(): + for (sig, (depth, code)) in features.items(): psig = _parseFeatureSignatureAndRewrite(sig) # file successfully parsed diff --git a/analyses/general.py b/analyses/general.py index 24a8374..f5ba851 100644 --- a/analyses/general.py +++ b/analyses/general.py @@ -166,7 +166,7 @@ def dictinvert(d): values into a dictionary that maps the values to the corresponding set of former keys.""" inv = dict() - for (k, v) in d.iteritems(): + for k, v in d.items(): for value in v: keys = inv.setdefault(value, []) keys.append(k) @@ -380,7 +380,7 @@ def _rewriteTwo(param): operand = __string | __hexadec | __integer | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') - expr = pypa.operatorPrecedence(operand, [ + expr = pypa.infixNotation(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), @@ -921,14 +921,14 @@ def _getFeatureStats(features): lofmean = 0 lofstd = 0 nof = len(features.keys()) - tmp = [item for (_, item) in features.itervalues()] + tmp = [item for (_, item) in list(features.values())] tmp = _flatten(tmp) floflist = list(map(lambda n: n.count('\n'), tmp)) if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = list(reduce(lambda m, n: m + n, floflist)) + lof = reduce(lambda m, n: m + n, floflist) lofmean = np.mean(floflist) if len(floflist) > 1: @@ -939,7 +939,7 @@ def _getFeatureStats(features): def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = list(filter(lambda t: t[1][0] == 1, features.iteritems())) # t = (sig, (depth, code)) + nof1 = list(filter(lambda t: t[1][0] == 1, features.items())) # t = (sig, (depth, code)) return nof1 @@ -983,7 +983,7 @@ def _compareFeatureCode(fcode): hom = {} hethom = {} - for (key, (_, item)) in features.iteritems(): + for (key, (_, item)) in features.items(): # distinguish according to feature-signature # shared code if '||' in key and ('&&' not in key): @@ -1123,7 +1123,7 @@ def _getGranularityStats(fcodetags): def __getNumOfFilesPerFeatureStats(filetofeatureconstants): featureconstantstofiles = dictinvert(filetofeatureconstants) - numbers = list(map(lambda v: len(v), featureconstantstofiles.values())) + numbers = list(map(lambda v: len(v), list(featureconstantstofiles.values()))) # mean if len(numbers) > 0: @@ -1184,7 +1184,7 @@ def apply(folder, options): def _mergeFeatures(ffeatures): """This function merges the, with the parameter given dictionary (ffeatures) to the afeatures (overall-features).""" - for (sig, (depth, code)) in ffeatures.iteritems(): + for (sig, (depth, code)) in ffeatures.items(): psig = _parseFeatureSignatureAndRewrite(sig) try: @@ -1340,7 +1340,7 @@ def _mergeFeatures(ffeatures): _getFeatureStats(afeatures) # SDEG + TDEG - sigs = _flatten(sigmap.values()) + sigs = _flatten(list(sigmap.values())) defs = list(__defset) (sdegmean, sdegstd, tdegmean, tdegstd) = \ _getScatteringTanglingDegrees(sigs, defs) diff --git a/analyses/generalvalues.py b/analyses/generalvalues.py index 3d4f688..e0d0443 100644 --- a/analyses/generalvalues.py +++ b/analyses/generalvalues.py @@ -126,7 +126,7 @@ def _prologCSV(folder, file, headings, delimiter=","): def _flatten(l): - """This function takes a list as input and returns a flatten version + """This function takes a list as input and returns a flattened version of the list. So all nested lists are unpacked and moved up to the level of the list.""" i = 0 @@ -732,7 +732,7 @@ def apply(folder, options): def _mergeFeatures(ffeatures): """This function merges the, with the parameter given dictionary (ffeatures) to the afeatures (overall-features).""" - for (sig, (depth, code)) in ffeatures.iteritems(): + for (sig, (depth, code)) in ffeatures.items(): psig = _parseFeatureSignatureAndRewrite(sig) try: @@ -778,7 +778,7 @@ def _mergeFeatures(ffeatures): # remove #else branches from list of features as there is no existing signature in the source code! if not options.rewriteifdefs: features = OrderedDict((sig, value) - for sig, value in features.iteritems() + for sig, value in features.items() if not sig.startswith(_elsePrefix)) # merge features of file in the global list of features @@ -792,7 +792,7 @@ def _mergeFeatures(ffeatures): print('INFO: parsing file (%5d) of (%5d) -- (%s).' % (fcount, ftotal, os.path.join(folder, file))) # get signatures and defines - sigs = _flatten(sigmap.values()) + sigs = _flatten(list(sigmap.values())) defs = list(__defset) # preparation: opn file for writing diff --git a/analyses/interaction.py b/analyses/interaction.py index 34d64ad..fa57e5f 100644 --- a/analyses/interaction.py +++ b/analyses/interaction.py @@ -763,14 +763,14 @@ def _getFeatureStats(features): lofmean = 0 lofstd = 0 nof = len(features.keys()) - tmp = [item for (_, item) in features.itervalues()] + tmp = [item for (_, item) in list(features.values())] tmp = _flatten(tmp) floflist = list(map(lambda n: n.count('\n'), tmp)) if len(floflist): lofmin = min(floflist) lofmax = max(floflist) - lof = list(reduce(lambda m, n: m + n, floflist)) + lof = reduce(lambda m, n: m + n, floflist) lofmean = np.mean(floflist) if len(floflist) > 1: @@ -781,7 +781,7 @@ def _getFeatureStats(features): def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = list(filter(lambda t: t[1][0] == 1, features.iteritems())) # t = (sig, (depth, code)) + nof1 = list(filter(lambda t: t[1][0] == 1, features.items())) # t = (sig, (depth, code)) return nof1 @@ -825,7 +825,7 @@ def _compareFeatureCode(fcode): hom = {} hethom = {} - for (key, (_, item)) in features.iteritems(): + for (key, (_, item)) in features.items(): # distinguish according to feature-signature # shared code if '||' in key and ('&&' not in key): @@ -1008,7 +1008,7 @@ def apply(folder, options): def _mergeFeatures(ffeatures): """This function merges the, with the parameter given dictionary (ffeatures) to the afeatures (overall-features).""" - for (sig, (depth, code)) in ffeatures.iteritems(): + for (sig, (depth, code)) in ffeatures.items(): (mal, psig) = _parseFeatureSignature(sig) try: diff --git a/cppstats/cli.py b/cppstats/cli.py index a7f5d98..41ee073 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -158,7 +158,7 @@ def getOptions(kinds, step=steps.ALL): parser.add_argument_group("Possible Kinds of Analyses ".upper(), ", ".join(kinds.keys())) # add options for each analysis kind - for kind in kinds.values(): + for kind in list(kinds.values()): analysisPart = kind[1] analysisThread = analysis.getKinds().get(analysisPart) analysisThread.addCommandLineOptions(parser) @@ -170,7 +170,7 @@ def getOptions(kinds, step=steps.ALL): parser.add_argument_group("Possible Kinds of Analyses ".upper(), ", ".join(kinds.keys())) # add options for each analysis kind - for cls in kinds.values(): + for cls in list(kinds.values()): cls.addCommandLineOptions(parser) # PARSE OPTIONS diff --git a/cppstats/preparation.py b/cppstats/preparation.py index ab7f667..6d9c44b 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -108,7 +108,7 @@ def runBashCommand(command, shell=False, stdin=None, stdout=None): def replaceMultiplePatterns(replacements, infile, outfile): - with open(infile, "rb") as source: + with open(infile, "r") as source: with open(outfile, "w") as target: data = source.read() for pattern, replacement in replacements.items(): @@ -117,9 +117,9 @@ def replaceMultiplePatterns(replacements, infile, outfile): def stripEmptyLinesFromFile(infile, outfile): - with open(infile, "rb") as source: + with open(infile, "r") as source: with open(outfile, "w") as target: - for line in source: + for line in source.readlines(): if line.strip(): target.write(line) # TODO Read bytes and write strings?? From bdcec443c2f05237b16a25e2a4215d8944a9ab7d Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 4 Apr 2023 11:13:48 +0200 Subject: [PATCH 22/72] Debug discipline Debug discipline Debug discipline Debug discipline Debug discipline Debug discipline Debug discipline Debug discipline --- analyses/discipline.py | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/analyses/discipline.py b/analyses/discipline.py index 7eff868..5dd11e4 100755 --- a/analyses/discipline.py +++ b/analyses/discipline.py @@ -112,11 +112,9 @@ def __getIfdefAnnotations__(self, root): treeifdefs = list() for _, elem in etree.iterwalk(root): - ns, tag = DisciplinedAnnotations.__cpprens.match(elem.tag). \ - groups() + ns, tag = DisciplinedAnnotations.__cpprens.match(elem.tag).groups() - if ns == DisciplinedAnnotations.__cppnscpp \ - and tag in DisciplinedAnnotations.__conditionals: + if ns == DisciplinedAnnotations.__cppnscpp and tag in DisciplinedAnnotations.__conditionals: treeifdefs.append(elem) return treeifdefs @@ -142,8 +140,10 @@ def __createListFromTreeifdefs__(self, treeifdefs): return -1 workerlist[-1].append(nifdef) last = workerlist[-1] - getpairs = zip(last, last[1:]) - map(lambda i: listifdefs.append(list(i)), getpairs) + getpairs = list(zip(last, last[1:])) + # calling the map function alone is not sufficient + # the value needs to be assigned in order for the lambda to be executed + _ = list(map(lambda i: listifdefs.append(list(i)), getpairs)) workerlist = workerlist[:-1] else: print('[ERROR] ill-formed tag (%s) occured in line (%4s).' % (tag, nifdef.sourceline)) @@ -158,7 +158,7 @@ def __filterConditionalPreprocessorDirectives(self, listifdefs): # iterate annotated blocks by determining all siblings of the #ifdef and filter out preprocessor # annotated elements resultlist = list(filter(lambda t: t[0].getnext() != t[1], listifdefs)) # t = (ifdef, endif) - print('[INFO] before after: %s <-> %s' % (str(len(listifdefs)), str(len(resultlist)))) + print(f'[INFO] before after: {len(listifdefs)} <-> {len(resultlist)}') return resultlist PATTLS = 0 # 1 << 0 => 1 @@ -238,7 +238,7 @@ def __checkStrictPattern__(self, listifdefs): nodeendif = listcorifdef[1] func = nodeendif.getparent() - if func is not None and func.tag.split('}')[1] == 'function': + if func and func.tag.split('}')[1] == 'function': nodefuncsibs = [sib for sib in func.itersiblings(preceding=True)] if nodeifdef == nodefuncsibs[0]: if self.opts.verbose: @@ -407,7 +407,7 @@ def __checkElseIfPattern__(self, listifdefs): # else parent -> #ifdef nodeendif = listcorifdef[-1] thensib = nodeendif.getprevious() - if thensib is None: + if not thensib: listundisciplinedunknown.append(listcorifdef) continue if thensib.tag.split('}')[1] not in ['then']: @@ -538,61 +538,61 @@ def __checkDiscipline__(self, treeifdefs, file): # check TLS pattern, subset of sibling pattern if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATTLS): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkStrictTLSCUPattern__(listifdefs) + listdisciplined, listundisciplined = self.__checkStrictTLSCUPattern__(listifdefs) self.compilationunit += len(listdisciplined) self.disciplined += len(listdisciplined) # checking fd pattern (part of tls) listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkStrictTLSFDPattern__(listifdefs) + listdisciplined, listundisciplined = self.__checkStrictTLSFDPattern__(listifdefs) self.functiontype += len(listdisciplined) self.disciplined += len(listdisciplined) # checking ill-formed compilation unit pattern listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkStrictPattern__(listifdefs) + listdisciplined, listundisciplined = self.__checkStrictPattern__(listifdefs) self.compilationunit += len(listdisciplined) self.disciplined += len(listdisciplined) # check if-then pattern if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATIFTHEN): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkIfThenPattern__(listifdefs) + listdisciplined, listundisciplined = self.__checkIfThenPattern__(listifdefs) self.wrapperif += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check case pattern if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATCASE): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkCasePattern__(listifdefs) + listdisciplined, listundisciplined = self.__checkCasePattern__(listifdefs) self.conditionalcase += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check else-if pattern if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATELSEIF): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkElseIfPattern__(listifdefs) + listdisciplined, listundisciplined = self.__checkElseIfPattern__(listifdefs) self.conditionalelif += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check param pattern if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATPARAM): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkParameter__(listifdefs) + listdisciplined, listundisciplined = self.__checkParameter__(listifdefs) self.parameter += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check expression pattern if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATEXP): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkExpression__(listifdefs) + listdisciplined, listundisciplined = self.__checkExpression__(listifdefs) self.expression += len(listdisciplined) self.undisciplinedknown += len(listdisciplined) # check sibling pattern; check this late because pattern might match for others as well if self.opts.disc_all or self.opts.check & (1 << DisciplinedAnnotations.PATSIB): listifdefs = list(listundisciplined) - (listdisciplined, listundisciplined) = self.__checkSiblingPattern__(listifdefs) + listdisciplined, listundisciplined = self.__checkSiblingPattern__(listifdefs) self.siblings += len(listdisciplined) self.disciplined += len(listdisciplined) @@ -614,6 +614,7 @@ def checkFile(self, file): # get root of the xml and iterate over it root = tree.getroot() treeifdefs = self.__getIfdefAnnotations__(root) + try: self.__checkDiscipline__(treeifdefs, file) except: From 6645515d9e760f954f7698cbe313d4fb38518ab8 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 4 Apr 2023 11:21:16 +0200 Subject: [PATCH 23/72] Fix operatorPrecedence being renamed in several files --- analyses/derivative.py | 2 +- analyses/featurelocations.py | 2 +- analyses/general.py | 2 +- analyses/generalvalues.py | 2 +- analyses/interaction.py | 2 +- lib/cpplib.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/analyses/derivative.py b/analyses/derivative.py index 44ca4d1..a6073d1 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -236,7 +236,7 @@ def _addIdentifier2Mal(p): operand = __string | __hexadec | __function | __integer | __identifier.setParseAction(_addIdentifier2Mal) compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / % & | << >>') - expr = pypa.operatorPrecedence(operand, [ + expr = pypa.infixNotation(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), diff --git a/analyses/featurelocations.py b/analyses/featurelocations.py index 6e1e542..a3c8f94 100644 --- a/analyses/featurelocations.py +++ b/analyses/featurelocations.py @@ -299,7 +299,7 @@ def _rewriteTwo(param): operand = __string | __hexadec | __integer | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') - expr = pypa.operatorPrecedence(operand, [ + expr = pypa.infixNotation(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), diff --git a/analyses/general.py b/analyses/general.py index f5ba851..86f3175 100644 --- a/analyses/general.py +++ b/analyses/general.py @@ -300,7 +300,7 @@ def _rewriteTwo(param): operand = __hexadec | __integer | __string | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') - expr = pypa.operatorPrecedence(operand, [ + expr = pypa.infixNotation(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), diff --git a/analyses/generalvalues.py b/analyses/generalvalues.py index e0d0443..7b1ef8a 100644 --- a/analyses/generalvalues.py +++ b/analyses/generalvalues.py @@ -278,7 +278,7 @@ def _rewriteTwo(param): operand = __string | __hexadec | __integer | __function | __identifier compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / & | << >> %') - expr = pypa.operatorPrecedence(operand, [ + expr = pypa.infixNotation(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), diff --git a/analyses/interaction.py b/analyses/interaction.py index fa57e5f..b616224 100644 --- a/analyses/interaction.py +++ b/analyses/interaction.py @@ -236,7 +236,7 @@ def _addIdentifier2Mal(p): operand = __string | __hexadec | __function | __integer | __identifier.setParseAction(_addIdentifier2Mal) compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / % & | << >>') - expr = pypa.operatorPrecedence(operand, [ + expr = pypa.infixNotation(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), diff --git a/lib/cpplib.py b/lib/cpplib.py index 2173f8a..36e0472 100644 --- a/lib/cpplib.py +++ b/lib/cpplib.py @@ -88,7 +88,7 @@ def _rewriteTwo(param): operand = __string | __hexadec | __integer | __function | __identifier operators = pypa.oneOf('&& ||') # extend with furhter operators - expr = pypa.operatorPrecedence(operand, [ + expr = pypa.infixNotation(operand, [ ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), (operators, 2, pypa.opAssoc.LEFT, _rewriteTwo), From 8ad7215d9d4556086a6effe6341937ccfd3e6ef0 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 4 Apr 2023 11:24:52 +0200 Subject: [PATCH 24/72] Cast filter, map and zip objects to lists in generalvalues.py --- analyses/generalvalues.py | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/analyses/generalvalues.py b/analyses/generalvalues.py index 7b1ef8a..cd8f0c9 100644 --- a/analyses/generalvalues.py +++ b/analyses/generalvalues.py @@ -99,17 +99,12 @@ def returnFileNames(folder, extfilt=['.xml']): currentfolder = wqueue[0] wqueue = wqueue[1:] foldercontent = os.listdir(currentfolder) - tmpfiles = filter(lambda n: os.path.isfile( - os.path.join(currentfolder, n)), foldercontent) - tmpfiles = filter(lambda n: os.path.splitext(n)[1] in extfilt, - tmpfiles) - tmpfiles = map(lambda n: os.path.join(currentfolder, n), - tmpfiles) + tmpfiles = list(filter(lambda n: os.path.isfile(os.path.join(currentfolder, n)), foldercontent)) + tmpfiles = list(filter(lambda n: os.path.splitext(n)[1] in extfilt, tmpfiles)) + tmpfiles = list(map(lambda n: os.path.join(currentfolder, n), tmpfiles)) filesfound += tmpfiles - tmpfolders = filter(lambda n: os.path.isdir( - os.path.join(currentfolder, n)), foldercontent) - tmpfolders = map(lambda n: os.path.join(currentfolder, n), - tmpfolders) + tmpfolders = list(filter(lambda n: os.path.isdir(os.path.join(currentfolder, n)), foldercontent)) + tmpfolders = list(map(lambda n: os.path.join(currentfolder, n), tmpfolders)) wqueue += tmpfolders return filesfound @@ -269,7 +264,7 @@ def _rewriteTwo(param): return 'modp(' + param[0][0] + ',' + param[0][2] + ')' ret = ' ' + __pt[param[0][1]] + ' ' - ret = '(' + ret.join(map(str, param[0][0::2])) + ')' + ret = '(' + ret.join(list(map(str, param[0][0::2]))) + ')' if param[0][1] in ['<', '>', '<=', '>=', '!=', '==']: ret = '(true &and ' + ret + ')' @@ -387,7 +382,7 @@ def _parseAndAddDefine(node): except pypa.ParseException: return - iden = ''.join(map(str, res[0])) + iden = ''.join(list(map(str, res[0]))) expn = res[-1] para = res[1:-1] __macrofuncs[iden] = (para, expn) @@ -651,7 +646,7 @@ def _getNestingDepths(root): # print "%s %s: %s (max: %s)" % (tag, _getMacroSignature(elem), cncur, cnmax) if 0 < len(cnlist): - nnitmp = filter(lambda n: n > 0, cnlist) + nnitmp = list(filter(lambda n: n > 0, cnlist)) __nestedIfdefsLevels += nnitmp __nestingDepthsOfBranches += sighist @@ -674,14 +669,14 @@ def __add(x, y): tang = [0] * len(sigs) # signatures overall for d in defines: dre = re.compile(r'\b' + d + r'\b') # using word boundaries - vec = map(lambda s: not dre.search(s) is None, sigs) + vec = list(map(lambda s: not dre.search(s) is None, sigs)) scat.append(vec.count(True)) - tang = map(__add, tang, vec) + tang = list(map(__add, tang, vec)) # create dictionaries from sigs and defines and corresponding # scattering and tangling values - scatdict = zip(defines, scat) - tangdict = zip(sigs, tang) + scatdict = list(zip(defines, scat)) + tangdict = list(zip(sigs, tang)) return scatdict, tangdict @@ -808,19 +803,19 @@ def _mergeFeatures(ffeatures): tangs = sorted([x[1] for x in tangvalues]) stfrow[0] = "tangling" - tanglingstring = ';'.join(map(str, tangs)) + tanglingstring = ';'.join(list(map(str, tangs))) stfrow[1] = tanglingstring stfwriter.writerow(stfrow) stfrow[0] = "scattering" - scatteringstring = ';'.join(map(str, scats)) + scatteringstring = ';'.join(list(map(str, scats))) stfrow[1] = scatteringstring stfwriter.writerow(stfrow) # nesting values stfrow[0] = "nestedIfdefsLevels" - ndstring = ';'.join(map(str, __nestedIfdefsLevels)) + ndstring = ';'.join(list(map(str, __nestedIfdefsLevels))) stfrow[1] = ndstring stfwriter.writerow(stfrow) From cf98945203161da8dcaabc84e6d4ff60c2e21ad5 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 4 Apr 2023 12:00:05 +0200 Subject: [PATCH 25/72] Replace has_key with in in derivative --- analyses/derivative.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/analyses/derivative.py b/analyses/derivative.py index a6073d1..33fb7b0 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -447,7 +447,7 @@ def _wrapFeatureUp(features, featuresgrinner, fcode, flist, finner): finner = finner[:-1] # handle the feature code - if features.has_key(itsig): + if itsig in features: features[itsig][1].append(itcode) else: features[itsig] = (len(flist) + 1, [itcode]) From cfd2f7d0f1f0e31c655d21dcafa2df99b68dbfc9 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 4 Apr 2023 18:15:27 +0200 Subject: [PATCH 26/72] Debug derivative Debug derivative Debug derivative Debug derivative Debug derivative Debug derivative Debug derivative --- analyses/derivative.py | 21 ++++++++++----------- analyses/interaction.py | 21 ++++++++++++--------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/analyses/derivative.py b/analyses/derivative.py index 33fb7b0..955c9b6 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -75,8 +75,7 @@ __conditionals_elif = ['elif'] __conditionals_else = ['else'] __conditionals_endif = ['endif'] -__conditionals_all = __conditionals + __conditionals_elif + \ - __conditionals_else +__conditionals_all = __conditionals + __conditionals_elif + __conditionals_else __macro_define = ['define'] __macrofuncs = {} # functional macros like: "GLIBVERSION(2,3,4)", # used as "GLIBVERSION(x,y,z) 100*x+10*y+z" @@ -351,7 +350,7 @@ def _getFeatureSignature(condinhist): if tag == 'else': continue if tag in ['if', 'elif']: - fsig = '(' + fsig + ') && (' + fname + ')' + fsig = f'({fsig}) && ({fname})' continue return fsig @@ -375,7 +374,7 @@ def _getASTFuture(node): node itself.""" dess = [] - while node is not None: + while node: dess += [sib for sib in node.itersiblings(preceding=False)] node = node.getparent() @@ -774,7 +773,7 @@ def _getFeatureStats(features): lofmean = 0 lofstd = 0 nof = len(features.keys()) - tmp = [item for (_, item) in list(features.values())] + tmp = list(features.values()) tmp = _flatten(tmp) floflist = list(map(lambda n: n.count('\n'), tmp)) @@ -792,7 +791,7 @@ def _getFeatureStats(features): def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = list(filter(lambda t: t[1][0] == 1, features.items())) # t = (sig, (depth, code)) + nof1 = list(filter(lambda t: t[1][0] == 1, list(features.items()))) # t = (sig, (depth, code)) return nof1 @@ -836,7 +835,7 @@ def _compareFeatureCode(fcode): hom = {} hethom = {} - for key, (_, item) in features.items(): + for key, (_, item) in list(features.items()): # distinguish according to feature-signature # shared code if '||' in key and '&&' not in key: @@ -901,7 +900,7 @@ def __add(x, y): tang = [0] * len(sigs) # signatures overall for d in defines: dre = re.compile(r'\b' + d + r'\b') # using word boundaries - vec = list(map(lambda s: not dre.search(s) is None, sigs)) + vec = list(map(lambda s: dre.search(s) is not None, sigs)) scat.append(vec.count(True)) tang = list(map(__add, tang, vec)) @@ -1028,7 +1027,7 @@ def apply(folder): def _mergeFeatures(ffeatures): """This function merges the, with the parameter given dictionary (ffeatures) to the afeatures (overall-features).""" - for sig, (depth, code) in ffeatures.items(): + for sig, (depth, code) in list(ffeatures.items()): mal, psig = _parseFeatureSignature(sig) try: @@ -1079,7 +1078,7 @@ def _mergeFeatures(ffeatures): # filter annotations that do not have any c-code # filter annotations with less than 2 features - afeatureitems = list(filter(lambda t: t[1][2] != [''], afeatures.items())) # t = (a, (f, d, c)) + afeatureitems = list(filter(lambda t: t[1][2] != [''], list(afeatures.items()))) # t = (a, (f, d, c)) annotations = list(map(lambda t: (t[0], t[1][0]), afeatureitems)) # t = (a, (f, b, c)) annotations2andmore = list(filter(lambda t: len(t[1]) > 1, annotations)) # t = (a, f) annotationmap = dict() @@ -1094,7 +1093,7 @@ def _mergeFeatures(ffeatures): featurenames = set(reduce(set.union, annotations2andmore, set([]))) for i in featurenames: fd.write(i + '\n') - for a, f in annotationmap.items(): + for a, f in list(annotationmap.items()): fd.write(prettyPrintSet(f) + ';' + a + '\n') fd.close() diff --git a/analyses/interaction.py b/analyses/interaction.py index b616224..090602b 100644 --- a/analyses/interaction.py +++ b/analyses/interaction.py @@ -124,8 +124,11 @@ def returnFileNames(folder, extfilt=['.xml']): def uniqueItems(l): - l = sorted(l) - return list(k for k, _ in itertools.groupby(l)) + r = [] + for i in l: + if i not in r: + r.append(i) + return r def _flatten(l): @@ -781,7 +784,7 @@ def _getFeatureStats(features): def _getFeaturesDepthOne(features): """This function returns all features that have the depth of one.""" - nof1 = list(filter(lambda t: t[1][0] == 1, features.items())) # t = (sig, (depth, code)) + nof1 = list(filter(lambda t: t[1][0] == 1, list(features.items()))) # t = (sig, (depth, code)) return nof1 @@ -825,7 +828,7 @@ def _compareFeatureCode(fcode): hom = {} hethom = {} - for (key, (_, item)) in features.items(): + for (key, (_, item)) in list(features.items()): # distinguish according to feature-signature # shared code if '||' in key and ('&&' not in key): @@ -863,7 +866,7 @@ def _getNumOfDefines(defset): """ # basic operation of this function is to check __defset against # __macrofuncs - funcmacros = __macrofuncs.keys() + funcmacros = list(__macrofuncs.keys()) funcmacros = list(map(lambda n: n.split('(')[0], funcmacros)) funcmacros = set(funcmacros) @@ -1008,11 +1011,11 @@ def apply(folder, options): def _mergeFeatures(ffeatures): """This function merges the, with the parameter given dictionary (ffeatures) to the afeatures (overall-features).""" - for (sig, (depth, code)) in ffeatures.items(): + for (sig, (depth, code)) in list(ffeatures.items()): (mal, psig) = _parseFeatureSignature(sig) try: - sigmatch = _checkForEquivalentSig(sigmap.keys(), psig) + sigmatch = _checkForEquivalentSig(list(sigmap.keys()), psig) (tmpflag, tmpdepth, tmpcode) = \ afeatures[sigmap[sigmatch][0]] tmpdepth = min(tmpdepth, depth) @@ -1058,7 +1061,7 @@ def _mergeFeatures(ffeatures): # filter annotations that do not have any c-code # filter annotations with less than 3 features - afeatureitems = list(filter(lambda t: t[1][2] != [''], afeatures.items())) # t = (a, (f, d, c)) + afeatureitems = list(filter(lambda t: t[1][2] != [''], list(afeatures.items()))) # t = (a, (f, d, c)) annotations = list(map(lambda t: t[1][0], afeatureitems)) # t = (a, (flag, b, c)) annotations3andmore = list(filter(lambda a: len(a) > 2, annotations)) annotations3andmore = uniqueItems(annotations3andmore) @@ -1071,7 +1074,7 @@ def _mergeFeatures(ffeatures): for annotation in annotations3andmore: combinations = list(map(lambda s: set(s), list(itertools.combinations(annotation, 2)))) allcomb = len(combinations) - occcomblist = list(set()) # TODO: = [] ? + occcomblist = [] for combination in combinations: if combination in annotations: occcomblist.append(combination) From a67229ace0a72b0d5a608ce092fa453d6a0bc3bc Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 18 Apr 2023 23:23:38 +0200 Subject: [PATCH 27/72] Debug interaction Debug interaction Debug interaction --- analyses/interaction.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/analyses/interaction.py b/analyses/interaction.py index 090602b..4d1d936 100644 --- a/analyses/interaction.py +++ b/analyses/interaction.py @@ -239,13 +239,13 @@ def _addIdentifier2Mal(p): operand = __string | __hexadec | __function | __integer | __identifier.setParseAction(_addIdentifier2Mal) compoperator = pypa.oneOf('< > <= >= == !=') calcoperator = pypa.oneOf('+ - * / % & | << >>') - expr = pypa.infixNotation(operand, [ - ('defined', 1, pypa.opAssoc.RIGHT, _rewriteOne), - ('!', 1, pypa.opAssoc.RIGHT, _rewriteOne), - (calcoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), - (compoperator, 2, pypa.opAssoc.LEFT, _rewriteTwo), - ('&&', 2, pypa.opAssoc.LEFT, _rewriteTwo), - ('||', 2, pypa.opAssoc.LEFT, _rewriteTwo), + expr = pypa.infix_notation(operand, [ + ('defined', 1, pypa.OpAssoc.RIGHT, _rewriteOne), + ('!', 1, pypa.OpAssoc.RIGHT, _rewriteOne), + (calcoperator, 2, pypa.OpAssoc.LEFT, _rewriteTwo), + (compoperator, 2, pypa.OpAssoc.LEFT, _rewriteTwo), + ('&&', 2, pypa.OpAssoc.LEFT, _rewriteTwo), + ('||', 2, pypa.OpAssoc.LEFT, _rewriteTwo), ]) try: From 81fa27ac8f4fd3766b67cbc5d92b94c77c9bfac3 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 26 Apr 2023 14:43:09 +0200 Subject: [PATCH 28/72] Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug Debug --- analyses/derivative.py | 2 +- analyses/interaction.py | 10 +++++++--- cppstats.sh | 5 ++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/analyses/derivative.py b/analyses/derivative.py index 955c9b6..efa8d66 100644 --- a/analyses/derivative.py +++ b/analyses/derivative.py @@ -153,7 +153,7 @@ def _collectDefines(d): if __curfile in __defsetf: __defsetf[__curfile].add(d[0]) else: - __defsetf[__curfile] = {d[0]} + __defsetf[__curfile] = set([d[0]]) return d diff --git a/analyses/interaction.py b/analyses/interaction.py index 4d1d936..526d81b 100644 --- a/analyses/interaction.py +++ b/analyses/interaction.py @@ -158,7 +158,7 @@ def _collectDefines(d): if __curfile in __defsetf: __defsetf[__curfile].add(d[0]) else: - __defsetf[__curfile] = {d[0]} + __defsetf[__curfile] = set([d[0]]) return d @@ -1062,9 +1062,12 @@ def _mergeFeatures(ffeatures): # filter annotations that do not have any c-code # filter annotations with less than 3 features afeatureitems = list(filter(lambda t: t[1][2] != [''], list(afeatures.items()))) # t = (a, (f, d, c)) - annotations = list(map(lambda t: t[1][0], afeatureitems)) # t = (a, (flag, b, c)) + annotations = list(map(lambda t: t[1][0], afeatureitems)) # t = (a, (flag, b, c)) + print("\n\n".join(list(map(lambda x: str(x), afeatureitems)))) annotations3andmore = list(filter(lambda a: len(a) > 2, annotations)) + print(annotations3andmore) annotations3andmore = uniqueItems(annotations3andmore) + print("\n".join(sorted(list(map(lambda x: str(sorted(list(x))), annotations3andmore))))) annotations3andmore = list(map(lambda s: set(s), annotations3andmore)) relevantannotations = list() missingannotations = list() @@ -1078,7 +1081,7 @@ def _mergeFeatures(ffeatures): for combination in combinations: if combination in annotations: occcomblist.append(combination) - combfeatset = set(reduce(set.union, occcomblist, set())) + combfeatset = reduce(set.union, occcomblist, set()) if combfeatset.issuperset(annotation): relevantannotations.append((annotation, occcomblist)) else: @@ -1098,6 +1101,7 @@ def _mergeFeatures(ffeatures): for i in relevantannotations: fd.write(str(i) + "\n") + fd.write(f"total annotations: {len(annotations3andmore):5d}\n") fd.write(f"relevant pairwise annotations: {len(relevantannotations):5d}\n") fd.write(f"missing pairwise annotations: {len(missingannotations):5d}\n") diff --git a/cppstats.sh b/cppstats.sh index a76c8de..31ccbc8 100755 --- a/cppstats.sh +++ b/cppstats.sh @@ -25,7 +25,6 @@ pushd `dirname $0` > /dev/null CPPSTATS=`pwd` popd > /dev/null - cd ${CPPSTATS} -PYTHONPATH="$PYTHONPATH:$CPPSTATS/lib" -./cppstats/cppstats.py "$@" +export PYTHONPATH="$PYTHONPATH:$CPPSTATS" +./cppstats/cppstats.py "$@" \ No newline at end of file From a01e7164e50b2cd8de71489fce9d2846cd56a9f2 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Tue, 2 May 2023 20:14:00 +0200 Subject: [PATCH 29/72] Add documentation --- CHANGELOG.md | 93 +++++++++++ INSTALL.md | 5 +- README.md | 7 +- cppstats_test.py | 398 +++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 17 +- 5 files changed, 501 insertions(+), 19 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 cppstats_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..bf4309e --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,93 @@ +# Critical changes from porting to Python 3 + +### General +- Dependencies were updated. +- The dependency to `statlib` has been removed and replaced with `numpy`. + +### Interaction Analysis +- The results have been wrong in the Python 2.7.x version of cppstats because duplicate combinations have not been removed +when counting the combinations with three or more variables. This is fixed in the Python 3 release. + +### Testing +- Added a test script that can be used to compare the output of the Python 2 version with the output of the Python 3 version. + +To run the test script you need to prepare various things: +1. Clone a project you would like to run the analyses on. +2. Inside the project folder, create a folder named "source" and move everything into there. + This should look something like this: + +*Before:* +``` +my_project +|- folder1 +| |- file1.cpp +|- file1.cpp +|- file2.cpp +``` + +*After:* +``` +my_project +|- source +| |- folder1 +| | |- file1.cpp +| |- file1.cpp +| |- file2.cpp +``` + +3. Clone the `cppstats` repository. +4. Inside the `cppstats` project folder, create a Python 2 virtual environment and a Python 3 one (called `venv2` and `venv3` respectively). + Of course, both Python 2 and Python 3 need to be installed on your system. +```shell +python -m venv venv3 +virtualenv --python=python2.7 venv2 +``` + +5. Test that the virtual environments are indeed running the correct version. From your `cppstats` folder, run +```shell +source venv2/bin/activate +python --version +deactivate +``` +which should yield some Python 2.7.x version. Similarly, try out +```shell +source venv3/bin/activate +python --version +deactivate +``` +which should yield some Python 3 (ideally, at least Python 3.10) version. + +6. Install all necessary dependencies in both virtual environments (see `setup.py` and `README.md`) + +7. Clone the `statlib` repository into the `cppstats` folder. At this point the `cppstats` folder should look something like this: +``` +cppstats +|- analyses +|- cppstats +|- lib +|- preparations +|- scripts +|- statlib +|- venv2 +|- venv3 +|- cppstats.sh +... +``` + +8. You should now be ready to run the test script. Refer to the command line parameter `--help` for detailed descriptions of + the other parameters and options. + +The script will automatically pull the latest versions of `cppstats` for both Python 2 and 3 and also run the specified +analyses for both versions by using the correct virtual environments. +There is also no need for to change the `cppstats_input.txt` or to run `cppstats.sh` manually. + +### Results +The results are collected and compared. An output can be seen on the command line (with varying degrees of details) and the +original result files can be found in the `results` folder that is created besides this script. + +**Please note** that results for the interaction analysis are not going to match, i.e. this test **will fail**. +This is caused by the fact that a bug was fixed in the Python 3 version that causes the analysis to not double count particular +combinations of variables (see `CHANGELOG.md`). + + + diff --git a/INSTALL.md b/INSTALL.md index 36970a0..161a14e 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -26,13 +26,12 @@ Current tested version of srcML: 1. checkout repo ```bash - git clone https://github.com/clhunsen/cppstats.git + git clone https://github.com/se-sic/cppstats.git ``` 2. install needed libraries (+ all dependencies) ```bash - sudo apt-get install astyle # artistic style (code formatter) sudo apt-get install xsltproc # XSLT 1.0 command line processor sudo apt-get install libxml2 libxml2-dev # library for processing XML sudo apt-get install gcc # GNU compiler collection @@ -118,8 +117,6 @@ CYGWIN libxslt-devel (1.1.27-2) zlib (1.2.8-3) zlib-devel (1.2.8-3) - - All/Utils/ - astyle (2.03-1) - All/Devel/ gcc (4.7.3-1) + all dependencies you are prompted for diff --git a/README.md b/README.md index 1ece2eb..ff8eb68 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,6 @@ Details of the latest version can be found on the cppstats project site at GitHu ## System Requirements * srcML (http://www.srcml.org/) -* astyle (http://astyle.sourceforge.net/) * libxml2 (http://www.xmlsoft.org/) * xsltproc (http://xmlsoft.org/XSLT/xsltproc2.html) * gcc (https://gcc.gnu.org/) @@ -61,10 +60,8 @@ system. In detail, cppstats was successfully tested under: -* Ubuntu 12.04, Python 2.7.*, and -* Cygwin, Python 2.7.*. - -Right now, Python 3.x is **NOT** supported. +* Ubuntu 12.04, Python >= 3.10, and +* Cygwin, Python >= 3.10. ## Quick Start diff --git a/cppstats_test.py b/cppstats_test.py new file mode 100644 index 0000000..150c498 --- /dev/null +++ b/cppstats_test.py @@ -0,0 +1,398 @@ +import click +import os +import shutil +import subprocess +import math +from colorama import Fore, Style +from datetime import datetime + +ANALYSES = ["general", "generalvalues", "discipline", "featurelocations", "derivative", "interaction"] +OUTPUT_FILE = { + "general": ["cppstats"], + "generalvalues": ["merged_scattering_degrees", "merged_tangling_degrees", "metric_values", + "nesting_degrees_toplevel_branches"], + "discipline": ["cppstats_discipline"], + "featurelocations": ["cppstats_featurelocations", "listoffeatures"], + "derivative": ["cppstats_derivative"], + "interaction": ["cppstats_interaction"] +} + +INPUT_FILE_NAME = "cppstats_input.txt" + +PY2_BRANCH = "py2numpy" +PY3_BRANCH = "py3" +BRANCHES = {2: PY2_BRANCH, + 3: PY3_BRANCH} + +PY2_VENV = "venv2/bin/activate" +PY3_VENV = "venv3/bin/activate" +VENVS = {2: PY2_VENV, + 3: PY3_VENV} + + +def create_result_dir(path, name): + if not os.path.exists(path): + os.mkdir(path) + + results_path = os.path.join(path, f"{name}_{datetime.now()}") + os.mkdir(results_path) + return results_path + + +def git_reset(verbose, cppstats_path): + if verbose: + subprocess.run(f"cd {cppstats_path} ; git reset --hard", + shell=True, executable='/bin/bash') + else: + subprocess.run(f"cd {cppstats_path} ; git reset --hard", + shell=True, executable='/bin/bash', + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + +def run_python(analysis, results_path, project_path, cppstats_path, verbose, ver): + # checkout correct branch + if verbose: + print(f" {Fore.CYAN}Checking out branch and pulling for Python {ver} ({BRANCHES[ver]}){Style.RESET_ALL}") + + git_reset(verbose, cppstats_path) + + if verbose: + subprocess.run(f"cd {cppstats_path} ; git checkout {BRANCHES[ver]}", + shell=True, executable='/bin/bash') + subprocess.run(f"cd {cppstats_path} ; git pull", + shell=True, executable='/bin/bash') + else: + subprocess.run(f"cd {cppstats_path} ; git checkout {BRANCHES[ver]}", + shell=True, executable='/bin/bash', + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + subprocess.run(f"cd {cppstats_path} ; git pull", + shell=True, executable='/bin/bash', + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + # rewrite the input file + if verbose: + print(f" {Fore.CYAN}Writing input file{Style.RESET_ALL}") + with open(os.path.join(cppstats_path, INPUT_FILE_NAME), "w") as f: + f.write(project_path) + + # delete all temporary cppstats files and folders + if verbose: + print(f" {Fore.CYAN}Deleting temporary cppstats files and folders{Style.RESET_ALL}") + for filename in os.listdir(project_path): + if filename == "source": + continue + + file_path = os.path.join(project_path, filename) + try: + if os.path.isfile(file_path) or os.path.islink(file_path): + os.unlink(file_path)# + if verbose: + print(f" Deleting file {filename}") + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + if verbose: + print(f" Deleting folder {filename}") + except Exception as e: + print(f'Failed to delete {file_path}. Reason: {e}') + + # execute cppstats + if verbose: + print(f" {Fore.CYAN}Running cppstats{Style.RESET_ALL}") + if verbose: + subprocess.run(f"source {os.path.join(cppstats_path, VENVS[ver])} ; cd {cppstats_path} ; " + f"./cppstats.sh --kind {analysis} --list", + shell=True, executable='/bin/bash') + else: + subprocess.run(f"source {os.path.join(cppstats_path, VENVS[ver])} ; cd {cppstats_path} ; " + f"./cppstats.sh --kind {analysis} --list", + shell=True, executable='/bin/bash', + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + + # copy result files + if verbose: + print(f" {Fore.CYAN}Copying results{Style.RESET_ALL}") + + for filename in OUTPUT_FILE[analysis]: + csv_filename = os.path.join(project_path, filename + ".csv") + results_filename = os.path.join(results_path, f"result_py{ver}_{analysis}_{filename}.csv") + shutil.copy2(csv_filename, results_filename) + + # reset branch + if verbose: + print(f" {Fore.CYAN}Resetting branch '{BRANCHES[ver]}'{Style.RESET_ALL}") + + git_reset(verbose, cppstats_path) + + if verbose: + print(f" {Fore.GREEN}Done{Style.RESET_ALL}") + + +def do_analysis(analysis, results_path, project_path, cppstats_path, verbose): + analysis_path = os.path.join(results_path, f"analysis_{analysis}") + os.mkdir(analysis_path) + + print(f" Python 2 - {analysis}") + run_python(analysis, analysis_path, project_path, cppstats_path, verbose, 2) + + print(f" Python 3 - {analysis}") + run_python(analysis, analysis_path, project_path, cppstats_path, verbose, 3) + + +def parse_result_file(file, project_name): + # read file without newlines at the end + with open(file, "r") as f: + lines = list(map(lambda l: l[:-1], f.readlines())) + + # read the separator, it is written in the first line in double quotes, e.g. "," + if "sep=" in lines[0]: + sep = lines[0].split('=')[1][0] + else: + sep = ";" # see below; there is no discipline in the discipline analysis + + # filter out paths in the first column + result = [] + for line in lines: + fst = line.split(sep)[0] + if project_name in fst: + line = line.replace(fst, project_name) + result.append(line) + return lines + + +def compute_difference(analysis, results_path, verbose, project_name, file_py2, file_py3, filename, i, n): + match = True + line_with_col_names = 1 + # parse the two result files + + if verbose: + print(f" Comparing results: {filename} ({i}/{n})") + print(f" {Fore.CYAN}Reading result files{Style.RESET_ALL}") + result_py2 = parse_result_file(file_py2, project_name) + result_py3 = parse_result_file(file_py3, project_name) + + if verbose: + print(f" {Fore.CYAN}Comparing result files{Style.RESET_ALL}") + # compare them by length; if the two files have different amounts of lines, then the comparison fails + if len(result_py2) != len(result_py3): + print(f" {Fore.RED}Length mismatch{Style.RESET_ALL}") + print(f" Python 2 produced {len(result_py2)} lines") + print(f" Python 3 produced {len(result_py3)} lines") + match = False + + # get the separator char from the first line + if "sep=" in result_py2[0] and "sep=" in result_py3[0]: + sep_py2 = result_py2[0].split('=')[1][0] + sep_py3 = result_py3[0].split('=')[1][0] + if len(sep_py2) != len(sep_py3): + print(f" {Fore.RED}Separator mismatch{Style.RESET_ALL}") + print(f" Python 2 uses '{sep_py2}'") + print(f" Python 3 uses '{sep_py3}'") + match = False + + sep = sep_py2 + else: # in discipline (ironically...), there is no separator written at the top + sep = ";" + line_with_col_names = 0 + + # get number of columns + col_names = result_py2[line_with_col_names].split(sep) + py2_cols = len(result_py2[1].split(sep)) + py3_cols = len(result_py3[1].split(sep)) + # compare them by length; if the two files have different amounts of columns, then the comparison fails + if py2_cols != py3_cols: + print(f" {Fore.RED}Column mismatch{Style.RESET_ALL}") + print(f" Python 2 produced {py2_cols} columns") + print(f" Python 3 produced {py3_cols} columns") + match = False + + if match: + if (filename == "merged_tangling_degrees" or + filename == "merged_scattering_degrees" or + filename == "cppstats_derivative"): + match = line_subset_difference(analysis, results_path, verbose, project_name, result_py2, + result_py3, filename, line_with_col_names, py2_cols, sep, col_names) + else: + match = default_difference(analysis, results_path, verbose, project_name, result_py2, result_py3, filename, + line_with_col_names, py2_cols, sep, col_names) + + if match: + print(f" {Fore.GREEN}Success - {analysis} / {filename}: {Style.RESET_ALL}Results are matching") + else: + print(f" {Fore.RED}Fail - {analysis} / {filename}: {Style.RESET_ALL}Results are not matching") + + +def divide_line(line): + if ";" in line: + l = line.split(";") + values = l[0].split(",") + [l[1]] + values = sorted(values) + return values + else: + values = line.split(",") + values = sorted(values) + return values + + +def line_subset_difference(analysis, results_path, verbose, project_name, result_py2, result_py3, + filename, line_with_col_names, py2_cols, sep, col_names): + match = True + + # Transform both line lists into lists of sorted lists of line components: + # y,x,z;x&&z -> ["x", "x&&z", "y", "z"] + result_py2 = sorted(list(map(lambda line: divide_line(line), result_py2))) + result_py3 = sorted(list(map(lambda line: divide_line(line), result_py3))) + + if verbose: + for i in range(len(result_py2)): + print(f" {result_py2[i]}") + print(f" {result_py3[i]}\n") + + # first, check whether all lines from the py2 output are in the py3 output as well + for l in range(2, len(result_py2)): + line_py2 = result_py2[l] + + if line_py2 in result_py3: + if verbose: + print(f" {Fore.GREEN}Line match Py2 <= Py3{Style.RESET_ALL} " + f"{Fore.CYAN}Line {l + 1}{Style.RESET_ALL}") + else: + match = False + print(f" {Fore.RED}Line missing Py2 <= Py3{Style.RESET_ALL} " + f"{Fore.CYAN}Line {l + 1}{Style.RESET_ALL}") + + # first, check whether all lines from the py2 output are in the py3 output as well + for l in range(2, len(result_py3)): + line_py3 = result_py3[l] + + if line_py3 in result_py2: + if verbose: + print(f" {Fore.GREEN}Line match Py2 >= Py3{Style.RESET_ALL} " + f"{Fore.CYAN}Line {l + 1}{Style.RESET_ALL}") + else: + match = False + print(f" {Fore.RED}Line missing Py2 >= Py3{Style.RESET_ALL} " + f"{Fore.CYAN}Line {l + 1}{Style.RESET_ALL}") + + return match + + +def default_difference(analysis, results_path, verbose, project_name, result_py2, result_py3, filename, + line_with_col_names, py2_cols, sep, col_names): + match = True + + # compare them line by line + for l in range(1, len(result_py2)): + line_py2 = result_py2[l] + line_py3 = result_py3[l] + + # compare column by column + for c in range(0, py2_cols): + try: + col_py2 = line_py2.split(sep)[c] + except: + print(f" {Fore.RED}Column mismatch{Style.RESET_ALL}") + print(f" Line {l} in the output of Python 2 does not " + f"have enough columns ({c - 1} instead of {py2_cols})") + match = False + continue + try: + col_py3 = line_py3.split(sep)[c] + except: + print(f" {Fore.RED}Column mismatch{Style.RESET_ALL}") + print(f" Line {l} in the output of Python 3 does not " + f"have enough columns ({c - 1} instead of {py2_cols})") + match = False + continue + + # try to parse a float + try: + float_py2 = float(col_py2) + float_py3 = float(col_py3) + + if not math.isclose(float_py2, float_py3): + print(f" {Fore.RED}Float value mismatch{Style.RESET_ALL}") + print(f" {Fore.CYAN}Line {l + 1}, Column {c + 1} ({col_names[c]}){Style.RESET_ALL} - " + f"Python 2: {col_py2} - Python 3: {col_py3}") + match = False + else: + if verbose: + print(f" {Fore.GREEN}Float value match{Style.RESET_ALL}" + f" {Fore.CYAN}Line {l + 1}, Column {c + 1} ({col_names[c]}){Style.RESET_ALL} - " + f"Python 2: {col_py2} - Python 3: {col_py3}") + except: + # if the conversion to float fails, then compare strings + if col_py2 != col_py3: + print(f" {Fore.RED}Value mismatch{Style.RESET_ALL}") + print(f" {Fore.CYAN}Line {l + 1}, Column {c + 1} ({col_names[c]}){Style.RESET_ALL} - " + f"Python 2: '{col_py2}' - Python 3: '{col_py3}'") + match = False + else: + if verbose: + print(f" {Fore.GREEN}Value match{Style.RESET_ALL}" + f" {Fore.CYAN}Line {l + 1}, Column {c + 1} ({col_names[c]}){Style.RESET_ALL} - " + f"Python 2: '{col_py2}' - Python 3: '{col_py3}'") + + return match + + +@click.command() +@click.option("-v", "--verbose", "verbose", + is_flag=True, + show_default=True, + default=False, + help="Print more output.") +@click.option("-n", "--notests", "notests", + is_flag=True, + show_default=True, + default=False, + help="If set, the tests are not executed.") +@click.option("-a", "--analysis", "analysis", + default="general", + type=click.Choice(["all"] + ANALYSES), + help="The kind of analysis to execute.") +@click.option("-r", "--result-path", "result_path", + default="results", + type=click.Path(exists=False, file_okay=False, dir_okay=True, readable=True), + help="The path where the results are to be stored.") +@click.option("-p", "--project-path", "project_path", + type=click.Path(exists=True, file_okay=False, dir_okay=True, readable=True), + help="The path where the project lies.", + required=True) +@click.option("-c", "--cppstats-path", "cppstats_path", + default="../cppstats", + type=click.Path(exists=False, file_okay=False, dir_okay=True, readable=True), + help="The path where cppstats lies.") +def main(analysis, result_path, project_path, cppstats_path, verbose, notests): + analyses = [analysis] if analysis != "all" else ANALYSES + + # delete trailing / from path + if project_path[-1] == "/": + project_path = project_path[:-2] + + project_name = project_path.split("/")[-1] + result_path = create_result_dir(result_path, project_name) + + i = 1 + for a in analyses: + print(f"Running analysis {analysis} on project '{project_name}' ({i}/{len(analyses)})") + do_analysis(a, result_path, project_path, cppstats_path, verbose) + + if notests: + continue + + j = 1 + for filename in OUTPUT_FILE[a]: + out_py2 = os.path.join(result_path, f"analysis_{a}", f"result_py2_{a}_{filename}.csv") + out_py3 = os.path.join(result_path, f"analysis_{a}", f"result_py3_{a}_{filename}.csv") + compute_difference(a, result_path, verbose, project_name, out_py2, out_py3, filename, j, + len(OUTPUT_FILE[a])) + j += 1 + i += 1 + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 2bf3b85..0668239 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ # Contributors: # Claus Hunsen # Andreas Ringlstetter +# Niklas Schneider from setuptools import setup, find_packages @@ -35,21 +36,17 @@ description='toolsuite for analyzing preprocessor-based software product lines', package_data={ - 'scripts' : ['*.sh'], - 'preparations' : ['*.xsl'] + 'scripts': ['*.sh'], + 'preparations': ['*.xsl'] }, install_requires=[ - #'statlib==1.2', - #'pyparsing==2.*', - #'enum34', - #'lxml>=3.4' - #'cpplib' # TODO This was missing but required + 'pyparsing', + 'enum34', + 'lxml>=3.4' ], - dependency_links=[ - 'https://github.com/clhunsen/python-statlib/archive/release-1.2.tar.gz#egg=statlib-1.2' - ], + dependency_links=[], entry_points={'console_scripts': [ 'cppstats = cppstats.cppstats:main', From 80889186ca87822484f65eb5036e1329afd390ad Mon Sep 17 00:00:00 2001 From: nlschn <73830746+nlschn@users.noreply.github.com> Date: Wed, 28 Jun 2023 14:21:44 +0200 Subject: [PATCH 30/72] Debug --- cppstats/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/__init__.py b/cppstats/__init__.py index d70006c..f380d1e 100644 --- a/cppstats/__init__.py +++ b/cppstats/__init__.py @@ -24,4 +24,4 @@ # ################################################# # make version number available in setup.py -from cppstats import version +#from cppstats import version From f86a0ecd8aa0b9ff477d51f888078c57f148ab65 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:18:22 +0200 Subject: [PATCH 31/72] Make imports more relative than before --- cppstats/__init__.py | 1 - cppstats/analysis.py | 4 ++-- cppstats/cli.py | 6 +----- cppstats/cppstats.py | 6 +++--- cppstats/preparation.py | 2 +- scripts/partial_preprocessor.py | 2 +- scripts/reversecpp.py | 2 +- 7 files changed, 9 insertions(+), 14 deletions(-) diff --git a/cppstats/__init__.py b/cppstats/__init__.py index f380d1e..06e5f55 100644 --- a/cppstats/__init__.py +++ b/cppstats/__init__.py @@ -24,4 +24,3 @@ # ################################################# # make version number available in setup.py -#from cppstats import version diff --git a/cppstats/analysis.py b/cppstats/analysis.py index 4e0374d..82e7819 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -35,10 +35,10 @@ # ################################################# # imports from subfolders -import cli +from .cli import * # import different kinds of analyses -from analyses import general, generalvalues, discipline, featurelocations, derivative, interaction +from .analyses import general, generalvalues, discipline, featurelocations, derivative, interaction # ################################################# # global constants diff --git a/cppstats/cli.py b/cppstats/cli.py index 41ee073..46c3b91 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -29,11 +29,7 @@ # ################################################# # imports from subfolders - -# TODO Are those really not needed? -# import preparation -# import cppstats as cstats # import cppstats.py and avoid confusion with module -import analysis +from .analysis import * # ################################################# # external modules diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index 4c8323b..ccc4aed 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -32,9 +32,9 @@ import tempfile # for temporary files # import different kinds of analyses -import cli -import preparation -import analysis +from .cli import * +from .preparation import * +from .analysis import * # ################################################# # version number diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 6d9c44b..23d0668 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -38,7 +38,7 @@ # ################################################# # paths -import preparations +from .preparations import * def getPreparationScript(filename): diff --git a/scripts/partial_preprocessor.py b/scripts/partial_preprocessor.py index a6419c0..30feb4a 100755 --- a/scripts/partial_preprocessor.py +++ b/scripts/partial_preprocessor.py @@ -27,7 +27,7 @@ from optparse import OptionParser import sys -from lib.cpplib import _filterAnnotatedIfdefs +from .lib.cpplib import _filterAnnotatedIfdefs class PartialPreprocessor: diff --git a/scripts/reversecpp.py b/scripts/reversecpp.py index 1c98e21..f9b1104 100644 --- a/scripts/reversecpp.py +++ b/scripts/reversecpp.py @@ -153,7 +153,7 @@ def createVariants(self, symbols, fname): for configuration in itertools.product(range(2), repeat=len(symbols)): configuration = list(configuration) pairs = zip(configuration, symbols) - validpairs = filter(lambda (m, n): m != 0, pairs) + validpairs = filter(lambda t: t[0] != 0, pairs) if len(validpairs): validdefines = list(zip(*validpairs)[1]) From cd738abb4afc60f0cf9293626b9fefc7e0e3ae66 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:20:58 +0200 Subject: [PATCH 32/72] Fix more imports --- cppstats/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index 82e7819..c63c362 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -38,7 +38,7 @@ from .cli import * # import different kinds of analyses -from .analyses import general, generalvalues, discipline, featurelocations, derivative, interaction +from . import general, generalvalues, discipline, featurelocations, derivative, interaction # ################################################# # global constants From 5b7cbb7078fc56bfb40c0370321dcffaf1ebeb6b Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:23:27 +0200 Subject: [PATCH 33/72] Fix more imports --- cppstats/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index c63c362..e32fa88 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -38,7 +38,7 @@ from .cli import * # import different kinds of analyses -from . import general, generalvalues, discipline, featurelocations, derivative, interaction +from . import * #general, generalvalues, discipline, featurelocations, derivative, interaction # ################################################# # global constants From 391616fa54411d8fe80c29dc26aef7beb1a4ef3b Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:24:39 +0200 Subject: [PATCH 34/72] Fix more imports --- cppstats/preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 23d0668..38a141e 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -38,7 +38,7 @@ # ################################################# # paths -from .preparations import * +from . import * def getPreparationScript(filename): From d7a535994eebb0b241fb7e48adc0ec8c352c0860 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:25:47 +0200 Subject: [PATCH 35/72] Fix more imports --- cppstats/preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 38a141e..4991929 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -48,7 +48,7 @@ def getPreparationScript(filename): # ################################################# # imports from subfolders -import cli +from .cli import * # for rewriting of #ifdefs to "if defined(..)" # for turning multiline macros to oneliners From f61bcba3a06b20363f531e5c864c336769355e38 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:29:15 +0200 Subject: [PATCH 36/72] Fix more imports --- cppstats/analysis.py | 2 +- cppstats/cli.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index e32fa88..82e7819 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -38,7 +38,7 @@ from .cli import * # import different kinds of analyses -from . import * #general, generalvalues, discipline, featurelocations, derivative, interaction +from .analyses import general, generalvalues, discipline, featurelocations, derivative, interaction # ################################################# # global constants diff --git a/cppstats/cli.py b/cppstats/cli.py index 46c3b91..5437452 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -29,6 +29,7 @@ # ################################################# # imports from subfolders + from .analysis import * # ################################################# @@ -156,7 +157,7 @@ def getOptions(kinds, step=steps.ALL): # add options for each analysis kind for kind in list(kinds.values()): analysisPart = kind[1] - analysisThread = analysis.getKinds().get(analysisPart) + analysisThread = getKinds().get(analysisPart) analysisThread.addCommandLineOptions(parser) elif step == steps.PREPARATION: From 71f5eedf00c4ca0e87d8dbf373e67c10ed4161fe Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:30:41 +0200 Subject: [PATCH 37/72] Fix more imports --- cppstats/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index 82e7819..8bd3e47 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -38,7 +38,7 @@ from .cli import * # import different kinds of analyses -from .analyses import general, generalvalues, discipline, featurelocations, derivative, interaction +from ..analyses import general, generalvalues, discipline, featurelocations, derivative, interaction # ################################################# # global constants From f224707b7a37e3f62dd13fead9a2033d34a49cb1 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:40:53 +0200 Subject: [PATCH 38/72] Fix more imports --- cppstats.sh | 1 + cppstats/analysis.py | 2 +- setup.py | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cppstats.sh b/cppstats.sh index 31ccbc8..865440b 100755 --- a/cppstats.sh +++ b/cppstats.sh @@ -27,4 +27,5 @@ CPPSTATS=`pwd` popd > /dev/null cd ${CPPSTATS} export PYTHONPATH="$PYTHONPATH:$CPPSTATS" +echo $PYTHONPATH ./cppstats/cppstats.py "$@" \ No newline at end of file diff --git a/cppstats/analysis.py b/cppstats/analysis.py index 8bd3e47..82e7819 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -38,7 +38,7 @@ from .cli import * # import different kinds of analyses -from ..analyses import general, generalvalues, discipline, featurelocations, derivative, interaction +from .analyses import general, generalvalues, discipline, featurelocations, derivative, interaction # ################################################# # global constants diff --git a/setup.py b/setup.py index 0668239..63c9689 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,9 @@ name='cppstats', version="0.9.4", packages=find_packages(exclude=['scripts']), + package_dir={ + 'cppstats': '.', + }, url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From 7593373c2fca68fa8f314e73d139ebfe035683c2 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:43:53 +0200 Subject: [PATCH 39/72] Fix more imports --- cppstats.sh | 1 - setup.py | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cppstats.sh b/cppstats.sh index 865440b..31ccbc8 100755 --- a/cppstats.sh +++ b/cppstats.sh @@ -27,5 +27,4 @@ CPPSTATS=`pwd` popd > /dev/null cd ${CPPSTATS} export PYTHONPATH="$PYTHONPATH:$CPPSTATS" -echo $PYTHONPATH ./cppstats/cppstats.py "$@" \ No newline at end of file diff --git a/setup.py b/setup.py index 63c9689..85d1788 100644 --- a/setup.py +++ b/setup.py @@ -28,10 +28,9 @@ setup( name='cppstats', version="0.9.4", - packages=find_packages(exclude=['scripts']), - package_dir={ - 'cppstats': '.', - }, + packages=["cppstats"],#find_packages(exclude=['scripts']), + package_dir={'cppstats': '.',}, + namespace_packages=['cppstats'], url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From bc1813e23328bbacbe544c0285aa130dc5e61070 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:44:39 +0200 Subject: [PATCH 40/72] Fix more imports --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 85d1788..0cb5b87 100644 --- a/setup.py +++ b/setup.py @@ -28,9 +28,8 @@ setup( name='cppstats', version="0.9.4", - packages=["cppstats"],#find_packages(exclude=['scripts']), + packages=find_packages(exclude=['scripts']), package_dir={'cppstats': '.',}, - namespace_packages=['cppstats'], url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From 8d16ce31ebbe278008e1d3e9b03fda04ca34e183 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:45:41 +0200 Subject: [PATCH 41/72] Fix more imports --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 0cb5b87..0668239 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ name='cppstats', version="0.9.4", packages=find_packages(exclude=['scripts']), - package_dir={'cppstats': '.',}, url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From 24bb49aadbb6d80ce48a8a63a4d2893f3e556ec3 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:47:01 +0200 Subject: [PATCH 42/72] Fix more imports --- cppstats/cppstats.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index ccc4aed..bd45fae 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -31,6 +31,8 @@ from collections import OrderedDict # for ordered dictionaries import tempfile # for temporary files +print(os.getcwd()) + # import different kinds of analyses from .cli import * from .preparation import * From c64ad724b0501a961fd7739e6c37afb7f1c25e72 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:49:20 +0200 Subject: [PATCH 43/72] Fix more imports --- cppstats/analysis.py | 2 +- cppstats/cppstats.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index 82e7819..31f5e09 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -38,7 +38,7 @@ from .cli import * # import different kinds of analyses -from .analyses import general, generalvalues, discipline, featurelocations, derivative, interaction +from analyses import general, generalvalues, discipline, featurelocations, derivative, interaction # ################################################# # global constants diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index bd45fae..ccc4aed 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -31,8 +31,6 @@ from collections import OrderedDict # for ordered dictionaries import tempfile # for temporary files -print(os.getcwd()) - # import different kinds of analyses from .cli import * from .preparation import * From fefdc2f1178cad1e95032a6094f2ecf7cac33526 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:51:42 +0200 Subject: [PATCH 44/72] Fix more imports --- analyses/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/analyses/__init__.py b/analyses/__init__.py index 8f539d6..cd8ec75 100644 --- a/analyses/__init__.py +++ b/analyses/__init__.py @@ -18,3 +18,5 @@ # # Contributors: # Claus Hunsen + +from .version import * \ No newline at end of file From fcb558ab352136118a640bdeeb5f810a9e84a0b5 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:54:16 +0200 Subject: [PATCH 45/72] Fix more imports --- analyses/__init__.py | 1 - cppstats/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/analyses/__init__.py b/analyses/__init__.py index cd8ec75..10596d2 100644 --- a/analyses/__init__.py +++ b/analyses/__init__.py @@ -19,4 +19,3 @@ # Contributors: # Claus Hunsen -from .version import * \ No newline at end of file diff --git a/cppstats/__init__.py b/cppstats/__init__.py index 06e5f55..57e8fb9 100644 --- a/cppstats/__init__.py +++ b/cppstats/__init__.py @@ -23,4 +23,4 @@ # ################################################# # make version number available in setup.py - +from . import version From e3f8a37ca0e1dfdb193d4913fb50761292301ebc Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 11:55:22 +0200 Subject: [PATCH 46/72] Fix more imports --- cppstats/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cppstats/__init__.py b/cppstats/__init__.py index 57e8fb9..02cca36 100644 --- a/cppstats/__init__.py +++ b/cppstats/__init__.py @@ -22,5 +22,4 @@ # ################################################# -# make version number available in setup.py -from . import version +# make version number available in setup.py \ No newline at end of file From 6c10bfc8b68b6000b320008fc8bf9d64a4c7a827 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 12:03:54 +0200 Subject: [PATCH 47/72] Fix more imports --- cppstats/cli.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cppstats/cli.py b/cppstats/cli.py index 5437452..aa2223a 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -100,6 +100,10 @@ def getOptions(kinds, step=steps.ALL): parser = ArgumentParser(formatter_class=RawTextHelpFormatter) + from . import __version__ + parser.add_argument('--version', dest='version', action="version", help='shows the version number', + version=__version__) + # TODO see above # version (uses CppstatsVersionAction instead of 'version' as action) # parser.add_argument('--version', action=CppstatsVersionAction, version=cstats.version()) From 1762471e75c93693d3616fe3cd38fff4ac77f4ed Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 12:07:56 +0200 Subject: [PATCH 48/72] Fix more imports --- cppstats/cli.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cppstats/cli.py b/cppstats/cli.py index aa2223a..17b26ce 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -100,9 +100,8 @@ def getOptions(kinds, step=steps.ALL): parser = ArgumentParser(formatter_class=RawTextHelpFormatter) - from . import __version__ - parser.add_argument('--version', dest='version', action="version", help='shows the version number', - version=__version__) + from . import version + parser.add_argument('--version', dest='version', action="version", help='shows the version number', type=version) # TODO see above # version (uses CppstatsVersionAction instead of 'version' as action) From bddf5c9f05a3c0e106ea8cb3caa0e053ea4210c0 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 12:08:57 +0200 Subject: [PATCH 49/72] Fix more imports --- cppstats/cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cppstats/cli.py b/cppstats/cli.py index 17b26ce..a17334f 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -100,8 +100,7 @@ def getOptions(kinds, step=steps.ALL): parser = ArgumentParser(formatter_class=RawTextHelpFormatter) - from . import version - parser.add_argument('--version', dest='version', action="version", help='shows the version number', type=version) + parser.add_argument('--version', dest='version', action="version", help='shows the version number', version=__version__) # TODO see above # version (uses CppstatsVersionAction instead of 'version' as action) From 4fd7e20273d673923f53e9e9b57b780e27c0993c Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 12:10:51 +0200 Subject: [PATCH 50/72] Fix more imports --- cppstats/cli.py | 8 ++++++++ cppstats/cppstats.py | 7 ------- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cppstats/cli.py b/cppstats/cli.py index a17334f..b5a2482 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -43,6 +43,14 @@ __inputlist_default = "cppstats_input.txt" +# ################################################# +# version number +__version__ = "v0.9.4" + + +def version(): + return f"cppstats {__version__}" + # ################################################# # definition of cppstats steps diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index ccc4aed..f4303c1 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -36,13 +36,6 @@ from .preparation import * from .analysis import * -# ################################################# -# version number -__version__ = "v0.9.4" - - -def version(): - return f"cppstats {__version__}" # ################################################# From d94a884b60e490a1d5d0da387501b2248ba39046 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 5 Jul 2023 12:13:59 +0200 Subject: [PATCH 51/72] Fix more imports --- cppstats/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cppstats/cli.py b/cppstats/cli.py index b5a2482..b1dd879 100644 --- a/cppstats/cli.py +++ b/cppstats/cli.py @@ -45,11 +45,11 @@ # ################################################# # version number -__version__ = "v0.9.4" +__version__ = "cppstats v0.9.4" def version(): - return f"cppstats {__version__}" + return __version__ # ################################################# From eab13fff6d6860f477244a91a94a44c9daca9ce4 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 11:39:20 +0200 Subject: [PATCH 52/72] Debug --- cppstats/cppstats.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index f4303c1..4b98fae 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -33,10 +33,10 @@ # import different kinds of analyses from .cli import * -from .preparation import * -from .analysis import * - - +# from .preparation import * +# from .analysis import * +import preparation +import analysis # ################################################# # collection of analyses From d0ee1dd985bf400c9156b6fdfd71b4e7834cd26d Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 11:49:12 +0200 Subject: [PATCH 53/72] Debug --- cppstats/cppstats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index 4b98fae..201b01e 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -35,7 +35,7 @@ from .cli import * # from .preparation import * # from .analysis import * -import preparation +import .preparation import analysis # ################################################# From 52c202c99a168027d05263c0b61d3d34ce04c357 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 11:55:00 +0200 Subject: [PATCH 54/72] Debug --- cppstats/cppstats.py | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index 201b01e..4b98fae 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -35,7 +35,7 @@ from .cli import * # from .preparation import * # from .analysis import * -import .preparation +import preparation import analysis # ################################################# diff --git a/setup.py b/setup.py index 0668239..42bfc1e 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name='cppstats', version="0.9.4", - packages=find_packages(exclude=['scripts']), + packages=find_packages(["analyses", "cppstats", "lib", "preparations"]), url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From e89e540f0d10dd3cbda68299b6cdb87da50ddba3 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 11:56:59 +0200 Subject: [PATCH 55/72] Debug --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 42bfc1e..93c17ae 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name='cppstats', version="0.9.4", - packages=find_packages(["analyses", "cppstats", "lib", "preparations"]), + packages=find_packages(["analyses", "cppstats_main", "lib", "preparations"]), url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From c642c0e11528bae6502b8275a6cc9514f8277b8b Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 11:58:56 +0200 Subject: [PATCH 56/72] Debug --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 93c17ae..da1a4fd 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ name='cppstats', version="0.9.4", packages=find_packages(["analyses", "cppstats_main", "lib", "preparations"]), + package_dir = {"": "cppstats"}, url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From a243858acf4bf2e0bd56b8400372b1a721b27cdd Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:00:26 +0200 Subject: [PATCH 57/72] Debug --- cppstats/cppstats.py | 4 ++-- setup.py | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index 4b98fae..69874c7 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -35,8 +35,8 @@ from .cli import * # from .preparation import * # from .analysis import * -import preparation -import analysis +import cppstats.preparation +import cppstats.analysis # ################################################# # collection of analyses diff --git a/setup.py b/setup.py index da1a4fd..42bfc1e 100644 --- a/setup.py +++ b/setup.py @@ -28,8 +28,7 @@ setup( name='cppstats', version="0.9.4", - packages=find_packages(["analyses", "cppstats_main", "lib", "preparations"]), - package_dir = {"": "cppstats"}, + packages=find_packages(["analyses", "cppstats", "lib", "preparations"]), url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From 505c73cc285198ecb1e1a3f503d77cd6d1ad1058 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:02:15 +0200 Subject: [PATCH 58/72] Debug --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 42bfc1e..3bbd640 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( name='cppstats', version="0.9.4", - packages=find_packages(["analyses", "cppstats", "lib", "preparations"]), + packages=find_packages(exclude=["scripts"]), url='http://www.fosd.net/cppstats', license='LGPLv3', author='Claus Hunsen', From 955d3bf64354018411e22e4e9c2815fecc50f5e1 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:03:27 +0200 Subject: [PATCH 59/72] Debug --- cppstats/cppstats.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index 69874c7..bc1649b 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -32,9 +32,7 @@ import tempfile # for temporary files # import different kinds of analyses -from .cli import * -# from .preparation import * -# from .analysis import * +import cppstats.cli import cppstats.preparation import cppstats.analysis From 7d23d274ae4907a265c994b9e2e060542d3de0e2 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:05:03 +0200 Subject: [PATCH 60/72] Debug --- cppstats/analysis.py | 2 +- cppstats/cppstats.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index 31f5e09..5878a2c 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -35,7 +35,7 @@ # ################################################# # imports from subfolders -from .cli import * +import cppstast.cli as cli # import different kinds of analyses from analyses import general, generalvalues, discipline, featurelocations, derivative, interaction diff --git a/cppstats/cppstats.py b/cppstats/cppstats.py index bc1649b..95cd13a 100755 --- a/cppstats/cppstats.py +++ b/cppstats/cppstats.py @@ -32,9 +32,9 @@ import tempfile # for temporary files # import different kinds of analyses -import cppstats.cli -import cppstats.preparation -import cppstats.analysis +import cppstats.cli as cli +import cppstats.preparation as preparation +import cppstats.analysis as analysis # ################################################# # collection of analyses From 027fd8ddded8c0118483952d9d9c0e061c681443 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:06:52 +0200 Subject: [PATCH 61/72] Debug --- cppstats/analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/analysis.py b/cppstats/analysis.py index 5878a2c..2032c16 100755 --- a/cppstats/analysis.py +++ b/cppstats/analysis.py @@ -35,7 +35,7 @@ # ################################################# # imports from subfolders -import cppstast.cli as cli +import cppstats.cli as cli # import different kinds of analyses from analyses import general, generalvalues, discipline, featurelocations, derivative, interaction From 0ffa4f4569c2e6b8e89d804390e5cafd6937071e Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:09:13 +0200 Subject: [PATCH 62/72] Debug --- cppstats/preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 4991929..8fa9511 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -53,7 +53,7 @@ def getPreparationScript(filename): # for rewriting of #ifdefs to "if defined(..)" # for turning multiline macros to oneliners # for deletion of include guards in H files -from preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards +from cppstats.preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards from lib import cpplib From 7ce9a5940649183f7791a93da70320dc36bf20bd Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:09:21 +0200 Subject: [PATCH 63/72] Debug --- cppstats/preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 8fa9511..d514918 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -55,7 +55,7 @@ def getPreparationScript(filename): # for deletion of include guards in H files from cppstats.preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards -from lib import cpplib +from cppstats.lib import cpplib # ################################################# # global constants From 8857d1e4af2120c38bb5e6b0c16552acba374ff4 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:12:03 +0200 Subject: [PATCH 64/72] Debug --- cppstats/preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index d514918..262297f 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -38,7 +38,7 @@ # ################################################# # paths -from . import * +import cppstats.preparations as preparatins def getPreparationScript(filename): From 0022dff49c951861d4c5999af1a77258abdcdc12 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:14:08 +0200 Subject: [PATCH 65/72] Debug --- cppstats/preparation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 262297f..8d1679c 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -38,8 +38,8 @@ # ################################################# # paths -import cppstats.preparations as preparatins - +import cppstats.preparation as preparation +import preparations def getPreparationScript(filename): return os.path.join(os.path.dirname(preparations.__file__), filename) From 7556e28c38eca53e0da51260211b316994d96495 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:15:53 +0200 Subject: [PATCH 66/72] Debug --- cppstats/preparation.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 8d1679c..f0b8665 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -39,7 +39,13 @@ # paths import cppstats.preparation as preparation -import preparations + +# for rewriting of #ifdefs to "if defined(..)" +# for turning multiline macros to oneliners +# for deletion of include guards in H files +from cppstats.preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards, __file__ + +from cppstats.lib import cpplib def getPreparationScript(filename): return os.path.join(os.path.dirname(preparations.__file__), filename) @@ -50,12 +56,7 @@ def getPreparationScript(filename): from .cli import * -# for rewriting of #ifdefs to "if defined(..)" -# for turning multiline macros to oneliners -# for deletion of include guards in H files -from cppstats.preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards -from cppstats.lib import cpplib # ################################################# # global constants From 85ea6130e464b1c8f6f2ac055cafb34adff164c5 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:17:15 +0200 Subject: [PATCH 67/72] Debug --- cppstats/preparation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index f0b8665..efdc5c7 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -43,9 +43,9 @@ # for rewriting of #ifdefs to "if defined(..)" # for turning multiline macros to oneliners # for deletion of include guards in H files -from cppstats.preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards, __file__ +from .preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards, __file__ -from cppstats.lib import cpplib +from .lib import cpplib def getPreparationScript(filename): return os.path.join(os.path.dirname(preparations.__file__), filename) From 53a881a24c0b9fcc96105de120913b57b858e64e Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:19:00 +0200 Subject: [PATCH 68/72] Debug --- cppstats/preparation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index efdc5c7..bfee001 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -43,7 +43,8 @@ # for rewriting of #ifdefs to "if defined(..)" # for turning multiline macros to oneliners # for deletion of include guards in H files -from .preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards, __file__ +from .preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards +import .preparations as preparations from .lib import cpplib From 3e80e8e8213955dcd8743ccf01b3a02c5de7d694 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:20:32 +0200 Subject: [PATCH 69/72] Debug --- cppstats/preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index bfee001..96bfddd 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -44,7 +44,7 @@ # for turning multiline macros to oneliners # for deletion of include guards in H files from .preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards -import .preparations as preparations +import preparations from .lib import cpplib From 8300ed43f3915957596e6240f061723f2fe76774 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:21:33 +0200 Subject: [PATCH 70/72] Debug --- cppstats/preparation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 96bfddd..43640a5 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -43,7 +43,7 @@ # for rewriting of #ifdefs to "if defined(..)" # for turning multiline macros to oneliners # for deletion of include guards in H files -from .preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards +from ..preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards import preparations from .lib import cpplib From c2df95c7fc4711c14a787ebf3f978e5c24be6f07 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:23:34 +0200 Subject: [PATCH 71/72] Debug --- cppstats/preparation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 43640a5..558e558 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -43,10 +43,12 @@ # for rewriting of #ifdefs to "if defined(..)" # for turning multiline macros to oneliners # for deletion of include guards in H files -from ..preparations import rewriteIfdefs, rewriteMultilineMacros, deleteIncludeGuards +import preparations.rewriteIfdefs +import preparations.rewriteMultilineMacros +import preparations.deleteIncludeGuards import preparations -from .lib import cpplib +import lib.cpplib def getPreparationScript(filename): return os.path.join(os.path.dirname(preparations.__file__), filename) From 8669dac7206adef2545b6aa36779668a0489c092 Mon Sep 17 00:00:00 2001 From: Niklas Schneider Date: Wed, 19 Jul 2023 12:24:47 +0200 Subject: [PATCH 72/72] Debug --- cppstats/preparation.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cppstats/preparation.py b/cppstats/preparation.py index 558e558..65d1260 100755 --- a/cppstats/preparation.py +++ b/cppstats/preparation.py @@ -43,9 +43,9 @@ # for rewriting of #ifdefs to "if defined(..)" # for turning multiline macros to oneliners # for deletion of include guards in H files -import preparations.rewriteIfdefs -import preparations.rewriteMultilineMacros -import preparations.deleteIncludeGuards +import preparations.rewriteIfdefs as rewriteIfdefs +import preparations.rewriteMultilineMacros as rewriteMultilineMacros +import preparations.deleteIncludeGuards as deleteIncludeGuards import preparations import lib.cpplib