More simplification of compiler pipeline.

- pycodegen: Remove little CodeGenerator classes
- pyassem:
  - Pull ComputeStackSize out of PyFlowGraph
  - Simplify instruction constants
  - Extract FlattenGraph

opy/compiler2/pyassem.py

@@ -8,6 +8,10 @@
from .consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS


HAS_JREL = set(dis.opname[op] for op in dis.hasjrel)
HAS_JABS = set(dis.opname[op] for op in dis.hasjabs)


# NOTE: Similar to ast.flatten().
def flatten(tup):
    elts = []
@@ -19,6 +23,69 @@ def flatten(tup):
    return elts


def ComputeStackDepth(graph):
    """Compute the max stack depth.

    Approach is to compute the stack effect of each basic block.
    Then find the path through the code with the largest total
    effect.
    """
    depth = {}
    exit = None
    for b in graph.getBlocks():
        depth[b] = findDepth(b.getInstructions())

    seen = {}

    def max_depth(b, d):
        if b in seen:
            return d
        seen[b] = 1
        d = d + depth[b]
        children = b.get_children()
        if children:
            return max([max_depth(c, d) for c in children])
        else:
            if not b.label == "exit":
                return max_depth(graph.exit, d)
            else:
                return d

    return max_depth(graph.entry, 0)


def FlattenGraph(blocks):
    insts = []
    pc = 0
    begin = {}
    end = {}
    for b in blocks:
        begin[b] = pc
        for inst in b.getInstructions():
            insts.append(inst)
            if len(inst) == 1:
                pc = pc + 1
            elif inst[0] != "SET_LINENO":
                # arg takes 2 bytes
                pc = pc + 3
        end[b] = pc
    pc = 0
    for i in range(len(insts)):
        inst = insts[i]
        if len(inst) == 1:
            pc = pc + 1
        elif inst[0] != "SET_LINENO":
            pc = pc + 3
        opname = inst[0]
        if opname in HAS_JREL:
            oparg = inst[1]
            offset = begin[oparg] - pc
            insts[i] = opname, offset
        elif opname in HAS_JABS:
            insts[i] = opname, begin[inst[1]]
    return insts


class FlowGraph(object):
    def __init__(self):
        self.current = self.entry = Block()
@@ -241,7 +308,7 @@ def get_followers(self):
        # Blocks that must be emitted *after* this one, because of
        # bytecode offsets (e.g. relative jumps) pointing to them.
        for inst in self.insts:
            if inst[0] in PyFlowGraph.hasjrel:
            if inst[0] in HAS_JREL:
                followers.add(inst[1])
        return followers

@@ -319,17 +386,16 @@ def setFreeVars(self, names):
    def setCellVars(self, names):
        self.cellvars = names

    def getCode(self):
    def getCode(self, stacksize):
        """Get a Python code object"""
        assert self.stage == RAW
        self.computeStackDepth()
        self.flattenGraph()
        assert self.stage == FLAT
        self.convertArgs()
        assert self.stage == CONV
        self.makeByteCode()
        assert self.stage == DONE
        return self.newCodeObject()
        return self.newCodeObject(stacksize)

    def dump(self, io=None):
        if io:
@@ -349,76 +415,12 @@ def dump(self, io=None):
        if io:
            sys.stdout = save

    def computeStackDepth(self):
        """Compute the max stack depth.

        Approach is to compute the stack effect of each basic block.
        Then find the path through the code with the largest total
        effect.
        """
        depth = {}
        exit = None
        for b in self.getBlocks():
            depth[b] = findDepth(b.getInstructions())

        seen = {}

        def max_depth(b, d):
            if b in seen:
                return d
            seen[b] = 1
            d = d + depth[b]
            children = b.get_children()
            if children:
                return max([max_depth(c, d) for c in children])
            else:
                if not b.label == "exit":
                    return max_depth(self.exit, d)
                else:
                    return d

        self.stacksize = max_depth(self.entry, 0)

    def flattenGraph(self):
        """Arrange the blocks in order and resolve jumps"""
        assert self.stage == RAW
        self.insts = insts = []
        pc = 0
        begin = {}
        end = {}
        for b in self.getBlocksInOrder():
            begin[b] = pc
            for inst in b.getInstructions():
                insts.append(inst)
                if len(inst) == 1:
                    pc = pc + 1
                elif inst[0] != "SET_LINENO":
                    # arg takes 2 bytes
                    pc = pc + 3
            end[b] = pc
        pc = 0
        for i in range(len(insts)):
            inst = insts[i]
            if len(inst) == 1:
                pc = pc + 1
            elif inst[0] != "SET_LINENO":
                pc = pc + 3
            opname = inst[0]
            if opname in self.hasjrel:
                oparg = inst[1]
                offset = begin[oparg] - pc
                insts[i] = opname, offset
            elif opname in self.hasjabs:
                insts[i] = opname, begin[inst[1]]
        self.insts = FlattenGraph(self.getBlocksInOrder())
        self.stage = FLAT

    hasjrel = set()
    for i in dis.hasjrel:
        hasjrel.add(dis.opname[i])
    hasjabs = set()
    for i in dis.hasjabs:
        hasjabs.add(dis.opname[i])

    def convertArgs(self):
        """Convert arguments from symbolic to concrete form"""
        assert self.stage == FLAT
@@ -544,7 +546,7 @@ def makeByteCode(self):
        opnum[dis.opname[num]] = num
    del num

    def newCodeObject(self):
    def newCodeObject(self, stacksize):
        assert self.stage == DONE
        if (self.flags & CO_NEWLOCALS) == 0:
            nlocals = 0
@@ -553,12 +555,13 @@ def newCodeObject(self):
        argcount = self.argcount
        if self.flags & CO_VARKEYWORDS:
            argcount = argcount - 1
        return types.CodeType(argcount, nlocals, self.stacksize, self.flags,
                        self.lnotab.getCode(), self.getConsts(),
                        tuple(self.names), tuple(self.varnames),
                        self.filename, self.name, self.lnotab.firstline,
                        self.lnotab.getTable(), tuple(self.freevars),
                        tuple(self.cellvars))
        return types.CodeType(
            argcount, nlocals, stacksize, self.flags,
            self.lnotab.getCode(), self.getConsts(),
            tuple(self.names), tuple(self.varnames),
            self.filename, self.name, self.lnotab.firstline,
            self.lnotab.getTable(), tuple(self.freevars),
            tuple(self.cellvars))

    def getConsts(self):
        """Return a tuple for the const slot of the code object

opy/compiler2/pycodegen.py

@@ -13,8 +13,6 @@
    CO_GENERATOR, CO_FUTURE_DIVISION,
    CO_FUTURE_ABSIMPORT, CO_FUTURE_WITH_STATEMENT, CO_FUTURE_PRINT_FUNCTION)

from .pyassem import TupleArg

callfunc_opcode_info = {
    # (Have *args, Have **args) : opcode
    (0,0) : "CALL_FUNCTION",
@@ -67,10 +65,10 @@ def compile(as_tree, filename, mode):
    elif mode == "exec":
        graph = pyassem.PyFlowGraph("<module>", as_tree.filename)
        futures = future.find_futures(as_tree)
        gen = ModuleCodeGenerator(futures, graph)
        gen = TopLevelCodeGenerator(graph, futures=futures)
    elif mode == "eval":
        graph = pyassem.PyFlowGraph("<expression>", as_tree.filename)
        gen = ExpressionCodeGenerator(graph)
        gen = TopLevelCodeGenerator(graph)
    else:
        raise ValueError("compile() 3rd arg must be 'exec' or "
                         "'eval' or 'single'")
@@ -84,7 +82,8 @@ def compile(as_tree, filename, mode):
    if mode == "single":
      gen.emit('RETURN_VALUE')

    return graph.getCode()
    stacksize = pyassem.ComputeStackDepth(graph)
    return graph.getCode(stacksize)


class LocalNameFinder(object):
@@ -206,7 +205,8 @@ def getCode(self):
        polymorphism of things that can be serialized to code objects.  Consts
        can be code objects!
        """
        return self.graph.getCode()
        stacksize = pyassem.ComputeStackDepth(self.graph)
        return self.graph.getCode(stacksize)

    def mangle(self, name):
        if self.class_name is not None:
@@ -1226,52 +1226,30 @@ def visitDict(self, node):
            self.emit('ROT_THREE')
            self.emit('STORE_SUBSCR')

class ModuleCodeGenerator(CodeGenerator):
    __super_init = CodeGenerator.__init__

class TopLevelCodeGenerator(CodeGenerator):
    scopes = None

    def __init__(self, futures, graph):
        self.futures = futures
    def __init__(self, graph, futures=None):
        self.graph = graph
        self.__super_init()

    def get_module(self):
        return self
        self.futures = futures or ()

class ExpressionCodeGenerator(CodeGenerator):
    __super_init = CodeGenerator.__init__

    scopes = None
    futures = ()

    def __init__(self, graph):
        self.graph = graph
        self.__super_init()
        # This is hacky: graph has to be initialized
        CodeGenerator.__init__(self)

    def get_module(self):
        return self

class InteractiveCodeGenerator(CodeGenerator):

    __super_init = CodeGenerator.__init__

    scopes = None
    futures = ()

    def __init__(self, graph):
        self.graph = graph
        self.__super_init()

    def get_module(self):
        return self
class InteractiveCodeGenerator(TopLevelCodeGenerator):

    def visitDiscard(self, node):
        # XXX Discard means it's an expression.  Perhaps this is a bad
        # name.
        self.visit(node.expr)
        self.emit('PRINT_EXPR')


class AbstractFunctionCode(object):
    optimized = 1
    lambdaCount = 0
@@ -1417,7 +1395,7 @@ def generateArgList(arglist):
        if isinstance(elt, str):
            args.append(elt)
        elif isinstance(elt, tuple):
            args.append(TupleArg(i * 2, elt))
            args.append(pyassem.TupleArg(i * 2, elt))
            extra.extend(misc.flatten(elt))
            count = count + 1
        else: