diff --git a/FWCore/ParameterSet/python/TreeCrawler.py b/FWCore/ParameterSet/python/TreeCrawler.py index 9118f086e98b8..821a320c089c2 100755 --- a/FWCore/ParameterSet/python/TreeCrawler.py +++ b/FWCore/ParameterSet/python/TreeCrawler.py @@ -23,7 +23,7 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -import sys, os, inspect, copy +import sys, os, inspect, copy, struct, dis, imp import modulefinder def packageNameFromFilename(name): @@ -107,7 +107,7 @@ def import_hook(self, name, caller=None, fromlist=None, level=-1): def import_module(self,partnam,fqname,parent): - if partnam in ("FWCore","os"): + if partnam in ("os","unittest"): r = None else: r = modulefinder.ModuleFinder.import_module(self,partnam,fqname,parent) @@ -126,6 +126,138 @@ def load_module(self, fqname, fp, pathname, (suffix, mode, type)): self._types[r.__name__] = type return r + def scan_opcodes_25(self, co, unpack = struct.unpack): + """ + This is basically just the default opcode scanner from ModuleFinder, but extended to also + look for "process.load()' commands. Since the Process object might not necassarily + be called "process", it scans for a call to a "load" method with a single parameter on + *any* object. If one is found it checks if the parameter is a string that refers to a valid + python module in the local or global area. If it does, the scanner assumes this was a call + to a Process object and yields the module name. + It's not possible to scan first for Process object declarations to get the name of the + objects since often (e.g. for customisation functions) the object is passed to a function + in a different file. + + The ModuleFinder.scan_opcodes_25 implementation this is based was taken from + https://hg.python.org/cpython/file/2.7/Lib/modulefinder.py#l364 + """ + # Scan the code, and yield 'interesting' opcode combinations + # Python 2.5 version (has absolute and relative imports) + code = co.co_code + names = co.co_names + consts = co.co_consts + LOAD_CONST = modulefinder.LOAD_CONST + IMPORT_NAME = modulefinder.IMPORT_NAME + STORE_OPS = modulefinder.STORE_OPS + HAVE_ARGUMENT = modulefinder.HAVE_ARGUMENT + LOAD_ATTR = chr(dis.opname.index('LOAD_ATTR')) + LOAD_NAME = chr(dis.opname.index('LOAD_NAME')) + CALL_FUNCTION = chr(dis.opname.index('CALL_FUNCTION')) + LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME + + try : + indexOfLoadConst = names.index("load") # This might throw a ValueError + # These are the opcodes required to access the "load" attribute. This might + # not even be a function, but I check for that later. + loadMethodOpcodes = LOAD_ATTR+struct.pack('=9 : # Need at least 9 codes for the full call + if code[:3]==loadMethodOpcodes : + # The attribute "load" is being accessed, need to make sure this is a function call. + # I'll look ahead and see if the CALL_FUNCTION code is used - this could be in a different + # place depending on the number of arguments, but I'm only interested in methods with a + # single argument so I know exactly where CALL_FUNCTION should be. + if code[6]==CALL_FUNCTION : + # I know this is calling a method called "load" with one argument. I need + # to find out what the argument is. Note that I still don't know if this is + # on a cms.Process object. + indexInTable=unpack('= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + +def removeRecursiveLoops( node, verbose=False, currentStack=None ) : + if currentStack is None : currentStack=[] + try : + duplicateIndex=currentStack.index( node ) # If there isn't a recursive loop this will raise a ValueError + if verbose : + print "Removing recursive loop in:" + for index in xrange(duplicateIndex,len(currentStack)) : + print " ",currentStack[index].name,"-->" + print " ",node.name + currentStack[-1].dependencies.remove(node) + except ValueError: + # No recursive loop found, so continue traversing the tree + currentStack.append( node ) + for subnode in node.dependencies : + removeRecursiveLoops( subnode, verbose, currentStack[:] ) def transformIntoGraph(depgraph,toplevel): packageDict = {} @@ -144,6 +276,7 @@ def transformIntoGraph(depgraph,toplevel): package = packageDict[key] package.dependencies = [packageDict[name] for name in value.keys() if name.count(".") == 2] + removeRecursiveLoops( packageDict[toplevel] ) # find and return the top level config return packageDict[toplevel]