| @@ -0,0 +1,23 @@ | ||
| # operation flags | ||
| OP_ASSIGN = 'OP_ASSIGN' | ||
| OP_DELETE = 'OP_DELETE' | ||
| OP_APPLY = 'OP_APPLY' | ||
| SC_LOCAL = 1 | ||
| SC_GLOBAL_IMPLICIT = 2 | ||
| SC_GLOBAL_EXPLICIT = 3 | ||
| SC_FREE = 4 | ||
| SC_CELL = 5 | ||
| SC_UNKNOWN = 6 | ||
| CO_OPTIMIZED = 0x0001 | ||
| CO_NEWLOCALS = 0x0002 | ||
| CO_VARARGS = 0x0004 | ||
| CO_VARKEYWORDS = 0x0008 | ||
| CO_NESTED = 0x0010 | ||
| CO_GENERATOR = 0x0020 | ||
| CO_GENERATOR_ALLOWED = 0 | ||
| CO_FUTURE_DIVISION = 0x2000 | ||
| CO_FUTURE_ABSIMPORT = 0x4000 | ||
| CO_FUTURE_WITH_STATEMENT = 0x8000 | ||
| CO_FUTURE_PRINT_FUNCTION = 0x10000 |
| @@ -0,0 +1,74 @@ | ||
| """Parser for future statements | ||
| """ | ||
| from compiler import ast, walk | ||
| def is_future(stmt): | ||
| """Return true if statement is a well-formed future statement""" | ||
| if not isinstance(stmt, ast.From): | ||
| return 0 | ||
| if stmt.modname == "__future__": | ||
| return 1 | ||
| else: | ||
| return 0 | ||
| class FutureParser: | ||
| features = ("nested_scopes", "generators", "division", | ||
| "absolute_import", "with_statement", "print_function", | ||
| "unicode_literals") | ||
| def __init__(self): | ||
| self.found = {} # set | ||
| def visitModule(self, node): | ||
| stmt = node.node | ||
| for s in stmt.nodes: | ||
| if not self.check_stmt(s): | ||
| break | ||
| def check_stmt(self, stmt): | ||
| if is_future(stmt): | ||
| for name, asname in stmt.names: | ||
| if name in self.features: | ||
| self.found[name] = 1 | ||
| else: | ||
| raise SyntaxError, \ | ||
| "future feature %s is not defined" % name | ||
| stmt.valid_future = 1 | ||
| return 1 | ||
| return 0 | ||
| def get_features(self): | ||
| """Return list of features enabled by future statements""" | ||
| return self.found.keys() | ||
| class BadFutureParser: | ||
| """Check for invalid future statements""" | ||
| def visitFrom(self, node): | ||
| if hasattr(node, 'valid_future'): | ||
| return | ||
| if node.modname != "__future__": | ||
| return | ||
| raise SyntaxError, "invalid future statement " + repr(node) | ||
| def find_futures(node): | ||
| p1 = FutureParser() | ||
| p2 = BadFutureParser() | ||
| walk(node, p1) | ||
| walk(node, p2) | ||
| return p1.get_features() | ||
| if __name__ == "__main__": | ||
| import sys | ||
| from compiler import parseFile, walk | ||
| for file in sys.argv[1:]: | ||
| print file | ||
| tree = parseFile(file) | ||
| v = FutureParser() | ||
| walk(tree, v) | ||
| print v.found | ||
| @@ -0,0 +1,73 @@ | ||
| def flatten(tup): | ||
| elts = [] | ||
| for elt in tup: | ||
| if isinstance(elt, tuple): | ||
| elts = elts + flatten(elt) | ||
| else: | ||
| elts.append(elt) | ||
| return elts | ||
| class Set: | ||
| def __init__(self): | ||
| self.elts = {} | ||
| def __len__(self): | ||
| return len(self.elts) | ||
| def __contains__(self, elt): | ||
| return elt in self.elts | ||
| def add(self, elt): | ||
| self.elts[elt] = elt | ||
| def elements(self): | ||
| return self.elts.keys() | ||
| def has_elt(self, elt): | ||
| return elt in self.elts | ||
| def remove(self, elt): | ||
| del self.elts[elt] | ||
| def copy(self): | ||
| c = Set() | ||
| c.elts.update(self.elts) | ||
| return c | ||
| class Stack: | ||
| def __init__(self): | ||
| self.stack = [] | ||
| self.pop = self.stack.pop | ||
| def __len__(self): | ||
| return len(self.stack) | ||
| def push(self, elt): | ||
| self.stack.append(elt) | ||
| def top(self): | ||
| return self.stack[-1] | ||
| def __getitem__(self, index): # needed by visitContinue() | ||
| return self.stack[index] | ||
| MANGLE_LEN = 256 # magic constant from compile.c | ||
| def mangle(name, klass): | ||
| if not name.startswith('__'): | ||
| return name | ||
| if len(name) + 2 >= MANGLE_LEN: | ||
| return name | ||
| if name.endswith('__'): | ||
| return name | ||
| try: | ||
| i = 0 | ||
| while klass[i] == '_': | ||
| i = i + 1 | ||
| except IndexError: | ||
| return name | ||
| klass = klass[i:] | ||
| tlen = len(klass) + len(name) | ||
| if tlen > MANGLE_LEN: | ||
| klass = klass[:MANGLE_LEN-tlen] | ||
| return "_%s%s" % (klass, name) | ||
| def set_filename(filename, tree): | ||
| """Set the filename attribute to filename on every node in tree""" | ||
| worklist = [tree] | ||
| while worklist: | ||
| node = worklist.pop(0) | ||
| node.filename = filename | ||
| worklist.extend(node.getChildNodes()) |
| @@ -0,0 +1,46 @@ | ||
| """Check for errs in the AST. | ||
| The Python parser does not catch all syntax errors. Others, like | ||
| assignments with invalid targets, are caught in the code generation | ||
| phase. | ||
| The compiler package catches some errors in the transformer module. | ||
| But it seems clearer to write checkers that use the AST to detect | ||
| errors. | ||
| """ | ||
| from compiler import ast, walk | ||
| def check(tree, multi=None): | ||
| v = SyntaxErrorChecker(multi) | ||
| walk(tree, v) | ||
| return v.errors | ||
| class SyntaxErrorChecker: | ||
| """A visitor to find syntax errors in the AST.""" | ||
| def __init__(self, multi=None): | ||
| """Create new visitor object. | ||
| If optional argument multi is not None, then print messages | ||
| for each error rather than raising a SyntaxError for the | ||
| first. | ||
| """ | ||
| self.multi = multi | ||
| self.errors = 0 | ||
| def error(self, node, msg): | ||
| self.errors = self.errors + 1 | ||
| if self.multi is not None: | ||
| print "%s:%s: %s" % (node.filename, node.lineno, msg) | ||
| else: | ||
| raise SyntaxError, "%s (%s:%s)" % (msg, node.filename, node.lineno) | ||
| def visitAssign(self, node): | ||
| # the transformer module handles many of these | ||
| pass | ||
| ## for target in node.nodes: | ||
| ## if isinstance(target, ast.AssList): | ||
| ## if target.lineno is None: | ||
| ## target.lineno = node.lineno | ||
| ## self.error(target, "can't assign to list comprehension") |
| @@ -0,0 +1,113 @@ | ||
| from compiler import ast | ||
| # XXX should probably rename ASTVisitor to ASTWalker | ||
| # XXX can it be made even more generic? | ||
| class ASTVisitor: | ||
| """Performs a depth-first walk of the AST | ||
| The ASTVisitor will walk the AST, performing either a preorder or | ||
| postorder traversal depending on which method is called. | ||
| methods: | ||
| preorder(tree, visitor) | ||
| postorder(tree, visitor) | ||
| tree: an instance of ast.Node | ||
| visitor: an instance with visitXXX methods | ||
| The ASTVisitor is responsible for walking over the tree in the | ||
| correct order. For each node, it checks the visitor argument for | ||
| a method named 'visitNodeType' where NodeType is the name of the | ||
| node's class, e.g. Class. If the method exists, it is called | ||
| with the node as its sole argument. | ||
| The visitor method for a particular node type can control how | ||
| child nodes are visited during a preorder walk. (It can't control | ||
| the order during a postorder walk, because it is called _after_ | ||
| the walk has occurred.) The ASTVisitor modifies the visitor | ||
| argument by adding a visit method to the visitor; this method can | ||
| be used to visit a child node of arbitrary type. | ||
| """ | ||
| VERBOSE = 0 | ||
| def __init__(self): | ||
| self.node = None | ||
| self._cache = {} | ||
| def default(self, node, *args): | ||
| for child in node.getChildNodes(): | ||
| self.dispatch(child, *args) | ||
| def dispatch(self, node, *args): | ||
| self.node = node | ||
| klass = node.__class__ | ||
| meth = self._cache.get(klass, None) | ||
| if meth is None: | ||
| className = klass.__name__ | ||
| meth = getattr(self.visitor, 'visit' + className, self.default) | ||
| self._cache[klass] = meth | ||
| ## if self.VERBOSE > 0: | ||
| ## className = klass.__name__ | ||
| ## if self.VERBOSE == 1: | ||
| ## if meth == 0: | ||
| ## print "dispatch", className | ||
| ## else: | ||
| ## print "dispatch", className, (meth and meth.__name__ or '') | ||
| return meth(node, *args) | ||
| def preorder(self, tree, visitor, *args): | ||
| """Do preorder walk of tree using visitor""" | ||
| self.visitor = visitor | ||
| visitor.visit = self.dispatch | ||
| self.dispatch(tree, *args) # XXX *args make sense? | ||
| class ExampleASTVisitor(ASTVisitor): | ||
| """Prints examples of the nodes that aren't visited | ||
| This visitor-driver is only useful for development, when it's | ||
| helpful to develop a visitor incrementally, and get feedback on what | ||
| you still have to do. | ||
| """ | ||
| examples = {} | ||
| def dispatch(self, node, *args): | ||
| self.node = node | ||
| meth = self._cache.get(node.__class__, None) | ||
| className = node.__class__.__name__ | ||
| if meth is None: | ||
| meth = getattr(self.visitor, 'visit' + className, 0) | ||
| self._cache[node.__class__] = meth | ||
| if self.VERBOSE > 1: | ||
| print "dispatch", className, (meth and meth.__name__ or '') | ||
| if meth: | ||
| meth(node, *args) | ||
| elif self.VERBOSE > 0: | ||
| klass = node.__class__ | ||
| if klass not in self.examples: | ||
| self.examples[klass] = klass | ||
| print self.visitor | ||
| print klass | ||
| for attr in dir(node): | ||
| if attr[0] != '_': | ||
| print "\t", "%-12.12s" % attr, getattr(node, attr) | ||
| return self.default(node, *args) | ||
| # XXX this is an API change | ||
| _walker = ASTVisitor | ||
| def walk(tree, visitor, walker=None, verbose=None): | ||
| if walker is None: | ||
| walker = _walker() | ||
| if verbose is not None: | ||
| walker.VERBOSE = verbose | ||
| walker.preorder(tree, visitor) | ||
| return walker.visitor | ||
| def dumpNode(node): | ||
| print node.__class__ | ||
| for attr in dir(node): | ||
| if attr[0] != '_': | ||
| print "\t", "%-10.10s" % attr, getattr(node, attr) |