More refactoring of compiler2.

pycodegen and pyassem:

- Move ComputeStackDepth back into PyFlowGraph.getCode().
- PyFlowGraph: extract setArgs(), which is only used by functions.

Lint fix: Remove unused imports from pgen2/driver.py

This change has caused regtest.sh change sin io.pyc and sh_spec.pyc. T
he io.pyc I assume is because because we no longer depend on io.  Not
sure about sh_spec.py.

I rebuilt Oil under OPy and ran the unit tests, and it works fine.

opy/compiler2/pyassem.py

@@ -46,14 +46,13 @@ def flatten(tup):

def getArgCount(args):
    n = len(args)
    if args:
        for arg in args:
            if isinstance(arg, TupleArg):
                n -= len(flatten(arg.names))
    for arg in args:
        if isinstance(arg, TupleArg):
            n -= len(flatten(arg.names))
    return n


def ComputeStackDepth(graph):
def ComputeStackDepth(blocks, entry_block, exit_block):
    """Compute the max stack depth.

    Approach is to compute the stack effect of each basic block.
@@ -62,7 +61,7 @@ def ComputeStackDepth(graph):
    """
    depth = {}
    exit = None
    for b in graph.getBlocks():
    for b in blocks:
        depth[b] = TRACKER.findDepth(b.getInstructions())

    seen = {}
@@ -77,11 +76,11 @@ def max_depth(b, d):
            return max([max_depth(c, d) for c in children])
        else:
            if not b.label == "exit":
                return max_depth(graph.exit, d)
                return max_depth(exit_block, d)
            else:
                return d

    return max_depth(graph.entry, 0)
    return max_depth(entry_block, 0)


def FlattenGraph(blocks):
@@ -217,7 +216,7 @@ def getContainedGraphs(self):
        return l


def order_blocks(start_block, exit_block):
def OrderBlocks(start_block, exit_block):
    """Order blocks so that they are emitted in the right order"""
    # Rules:
    # - when a block has a next block, the next block must be emitted just after
@@ -297,7 +296,7 @@ def __init__(self, label=''):
        Block._count += 1

    # BUG FIX: This is needed for deterministic order in sets (and dicts?).
    # See order_blocks() below.  remaining is set() of blocks.  If we rely on
    # See OrderBlocks() below.  remaining is set() of blocks.  If we rely on
    # the default id(), then the output bytecode is NONDETERMINISTIC.
    def __hash__(self):
        return self.bid
@@ -386,7 +385,7 @@ class PyFlowGraph(FlowGraph):
    code_object = Assemble(flow_graph)
    """

    def __init__(self, name, filename, args=(), optimized=0, klass=None):
    def __init__(self, name, filename, optimized=0, klass=None):
        """
        Args:
          klass: Whether we're compiling a class block.
@@ -396,8 +395,6 @@ def __init__(self, name, filename, args=(), optimized=0, klass=None):
        self.name = name  # name that is put in the code object
        self.filename = filename
        self.docstring = None
        self.args = args
        self.argcount = getArgCount(args)
        self.klass = klass
        if optimized:
            self.flags = CO_OPTIMIZED | CO_NEWLOCALS
@@ -417,10 +414,20 @@ def __init__(self, name, filename, args=(), optimized=0, klass=None):
        # The offsets used by LOAD_CLOSURE/LOAD_DEREF refer to both
        # kinds of variables.
        self.closure = []
        self.varnames = list(args) or []
        for i, var in enumerate(self.varnames):
            if isinstance(var, TupleArg):
                self.varnames[i] = var.getName()

        # Mutated by setArgs()
        self.varnames = []
        self.argcount = 0

    def setArgs(self, args):
        """Only called by functions, not modules or classes."""
        assert not self.varnames   # Nothing should have been added
        if args:
            self.varnames = list(args)
            for i, var in enumerate(self.varnames):
                if isinstance(var, TupleArg):
                    self.varnames[i] = var.getName()
            self.argcount = getArgCount(args)

    def setDocstring(self, doc):
        self.docstring = doc
@@ -440,7 +447,7 @@ def setFreeVars(self, names):
    def setCellVars(self, names):
        self.cellvars = names

    def getCode(self, stacksize):
    def getCode(self):
        """Assemble a Python code object."""
        # TODO: Split into two representations?  Graph and insts?

@@ -449,7 +456,8 @@ def getCode(self, stacksize):
        # up varnames.
        # Really we need a shared varnames representation.

        blocks = order_blocks(self.entry, self.exit)
        stacksize = ComputeStackDepth(self.blocks, self.entry, self.exit)
        blocks = OrderBlocks(self.entry, self.exit)
        insts = FlattenGraph(blocks)

        self.consts.insert(0, self.docstring)

opy/compiler2/pycodegen.py

@@ -86,8 +86,7 @@ def compile(as_tree, filename, mode):
    if mode == "single":
      gen.emit('RETURN_VALUE')

    stacksize = pyassem.ComputeStackDepth(graph)
    return graph.getCode(stacksize)
    return graph.getCode()


class LocalNameFinder(object):
@@ -193,8 +192,7 @@ def getCode(self):
        polymorphism of things that can be serialized to code objects.  Consts
        can be code objects!
        """
        stacksize = pyassem.ComputeStackDepth(self.graph)
        return self.graph.getCode(stacksize)
        return self.graph.getCode()

    def mangle(self, name):
        if self.class_name is not None:
@@ -338,6 +336,8 @@ def visitLambda(self, node):
        obj_name = "<lambda.%d>" % gLambdaCount
        gLambdaCount += 1

        # TODO: _GenerateArgList -> _AssertNoTupleArgs, instantiate graph, pass
        # args
        gen = FunctionCodeGenerator(node, self.scopes, obj_name,
                                    self.class_name, self.get_module())
        self._funcOrLambda(node, gen, 0, isLambda=True)
@@ -347,6 +347,7 @@ def _funcOrLambda(self, node, gen, ndecorators, isLambda=False):
        if not isLambda and node.doc:
            gen.setDocstring(node.doc)
        gen.FindLocals()

        walk(node.code, gen)
        gen.Finish(isLambda=isLambda)

@@ -1281,6 +1282,12 @@ def visitDiscard(self, node):
        self.emit('PRINT_EXPR')


# NOTE: This feature removed in Python 3!  I didn't even know about it!
#
# https://www.python.org/dev/peps/pep-3113/
# def fxn(a, (b, c), d):
#    pass

def _GenerateArgList(arglist):
    """Generate an arg list marking TupleArgs"""
    args = []
@@ -1312,8 +1319,9 @@ def __init__(self, func, scopes, obj_name, class_name, mod):
        self.module = mod

        self.args, self.hasTupleArg = _GenerateArgList(func.argnames)
        self.graph = pyassem.PyFlowGraph(obj_name, func.filename, self.args,
        self.graph = pyassem.PyFlowGraph(obj_name, func.filename,
                                         optimized=1)
        self.graph.setArgs(self.args)
        CodeGenerator.__init__(self)

    def get_module(self):

opy/compiler2/symbols.py

@@ -212,7 +212,7 @@ def __init__(self, name, module):

class SymbolVisitor(object):
    def __init__(self):
        self.scopes = {}
        self.scopes = {}  # The "return value" of walk()
        self.klass = None

    # node that define new scopes

opy/pgen2/driver.py

@@ -16,8 +16,6 @@
__all__ = ["Driver"]

# Python imports
import codecs
import io
import logging

# Pgen imports
@@ -40,10 +38,10 @@ def parse_tokens(self, tokens, start_symbol=None, debug=False):
        p.setup(start=start_symbol)
        lineno = 1
        column = 0
        type = value = start = end = line_text = None
        type_ = value = start = end = line_text = None
        prefix = ""
        for quintuple in tokens:
            type, value, start, end, line_text = quintuple
            type_, value, start, end, line_text = quintuple
            if start != (lineno, column):
                assert (lineno, column) <= start, ((lineno, column), start)
                s_lineno, s_column = start
@@ -54,19 +52,19 @@ def parse_tokens(self, tokens, start_symbol=None, debug=False):
                if column < s_column:
                    prefix += line_text[column:s_column]
                    column = s_column
            if type in (tokenize.COMMENT, tokenize.NL):
            if type_ in (tokenize.COMMENT, tokenize.NL):
                prefix += value
                lineno, column = end
                if value.endswith("\n"):
                    lineno += 1
                    column = 0
                continue
            if type == token.OP:
                type = grammar.opmap[value]
            if type_ == token.OP:
                type_ = grammar.opmap[value]
            if debug:
                self.logger.debug("%s %r (prefix=%r)",
                                  token.tok_name[type], value, prefix)
            if p.addtoken(type, value, (prefix, start)):
                                  token.tok_name[type_], value, prefix)
            if p.addtoken(type_, value, (prefix, start)):
                if debug:
                    self.logger.debug("Stop.")
                break
@@ -78,5 +76,5 @@ def parse_tokens(self, tokens, start_symbol=None, debug=False):
        else:
            # We never broke out -- EOF is too soon (how can this happen???)
            raise parse.ParseError("incomplete input",
                                   type, value, (prefix, start))
                                   type_, value, (prefix, start))
        return p.rootnode