pyassem: split up Frame and FlowGraph.

MakeCodeObject is now a free function that takes them both as input!  We
no longer have a big monolithic mutable object.

opy/compiler2/pyassem.py

@@ -205,7 +205,7 @@ def getContainedGraphs(self):
        return contained


class _FlowGraph(object):
class FlowGraph(object):

    def __init__(self):
        self.current = self.entry = Block()
@@ -273,13 +273,7 @@ def getContainedGraphs(self):
        return l


# Placeholder
class FlowGraph(object):
  pass


# TODO: Unify FlowGraph and PyFlowGraph.
class Frame(_FlowGraph):
class Frame(object):
    """Something that gets turned into a single code object.

    Code objects and consts are mutually recursive.
@@ -290,7 +284,6 @@ def __init__(self, name, filename, optimized=0, klass=None):
        Args:
          klass: Whether we're compiling a class block.
        """
        _FlowGraph.__init__(self)
        self.name = name  # name that is put in the code object
        self.filename = filename
        self.flags = (CO_OPTIMIZED | CO_NEWLOCALS) if optimized else 0
@@ -327,69 +320,72 @@ def setFlag(self, flag):
    def checkFlag(self, flag):
        return bool(self.flags & flag)

    def _ReorderCellVars(self):
        """Reorder cellvars so the ones in self.varnames are first.

        And prune from freevars (?)
        """
        lookup = set(self.cellvars)
        remaining = lookup - set(self.varnames)

        cellvars = [n for n in self.varnames if n in lookup]
        cellvars.extend(remaining)
        return cellvars

    def MakeCodeObject(self):
        """Order blocks, encode instructions, and create types.CodeType()."""

        # Compute stack depth per basic block.
        depths = {}
        for b in self.blocks:
            depths[b] = BLOCK_STACK_DEPTH.Sum(b.getInstructions())

        # Compute maximum stack depth for any control flow.
        g = GraphStackDepth(depths, self.exit)
        stacksize = g.Max(self.entry, 0)

        blocks = OrderBlocks(self.entry, self.exit)
        insts = FlattenGraph(blocks)

        cellvars = self._ReorderCellVars()
        consts = [self.docstring]
        names = []
        # The closure list is used to track the order of cell variables and
        # free variables in the resulting code object.  The offsets used by
        # LOAD_CLOSURE/LOAD_DEREF refer to both kinds of variables.
        closure = cellvars + self.freevars

        # Convert arguments from symbolic to concrete form.
        enc = ArgEncoder(self.klass, consts, names, self.varnames, closure)
        # Mutates not only insts, but also self.consts, self.names, etc.
        enc.Run(insts)

        if self.flags & CO_NEWLOCALS:
            nlocals = len(self.varnames)
        else:
            nlocals = 0

        if self.flags & CO_VARKEYWORDS:
            self.argcount -= 1
        if self.flags & CO_VARARGS:
            self.argcount -= 1
def _ReorderCellVars(frame):
    """Reorder cellvars so the ones in self.varnames are first.

        a = Assembler()
        bytecode, firstline, lnotab = a.Run(insts)

        return types.CodeType(
            self.argcount, nlocals, stacksize, self.flags,
            bytecode,
            tuple(consts),
            tuple(names),
            tuple(self.varnames),
            self.filename, self.name, firstline,
            lnotab,
            tuple(self.freevars),
            tuple(cellvars))
    And prune from freevars (?)
    """
    lookup = set(frame.cellvars)
    remaining = lookup - set(frame.varnames)

    cellvars = [n for n in frame.varnames if n in lookup]
    cellvars.extend(remaining)
    return cellvars


def MakeCodeObject(frame, graph):
    """Order blocks, encode instructions, and create types.CodeType()."""

    # Compute stack depth per basic block.
    depths = {}
    for b in graph.blocks:
        depths[b] = BLOCK_STACK_DEPTH.Sum(b.getInstructions())

    # Compute maximum stack depth for any control flow.
    g = GraphStackDepth(depths, graph.exit)
    stacksize = g.Max(graph.entry, 0)

    blocks = OrderBlocks(graph.entry, graph.exit)
    insts = FlattenGraph(blocks)

    cellvars = _ReorderCellVars(frame)
    consts = [frame.docstring]
    names = []
    # The closure list is used to track the order of cell variables and
    # free variables in the resulting code object.  The offsets used by
    # LOAD_CLOSURE/LOAD_DEREF refer to both kinds of variables.
    closure = cellvars + frame.freevars

    # Convert arguments from symbolic to concrete form.
    enc = ArgEncoder(frame.klass, consts, names, frame.varnames, closure)
    # Mutates not only insts, but also consts, names, etc.
    enc.Run(insts)

    if frame.flags & CO_NEWLOCALS:
        nlocals = len(frame.varnames)
    else:
        nlocals = 0

    argcount = frame.argcount
    if frame.flags & CO_VARKEYWORDS:
        argcount -= 1
    if frame.flags & CO_VARARGS:
        argcount -= 1

    a = Assembler()
    bytecode, firstline, lnotab = a.Run(insts)

    return types.CodeType(
        argcount, nlocals, stacksize, frame.flags,
        bytecode,
        tuple(consts),
        tuple(names),
        tuple(frame.varnames),
        frame.filename, frame.name, firstline,
        lnotab,
        tuple(frame.freevars),
        tuple(cellvars))


def _NameToIndex(name, L):
@@ -439,7 +435,7 @@ def Run(self, insts):
    def _convert_LOAD_CONST(self, arg):
        from . import pycodegen
        if isinstance(arg, pycodegen.CodeGenerator):
            arg = arg.frame.MakeCodeObject()
            arg = MakeCodeObject(arg.frame, arg.graph)
            # arg.code
        return _NameToIndex(arg, self.consts)

opy/compiler2/pycodegen.py

@@ -108,12 +108,11 @@ def __init__(self, frame, graph, ctx):
        self.last_lineno = None
        self._div_op = "BINARY_DIVIDE"

        # Set up methods
        self.emit = self.frame.emit
        self.newBlock = self.frame.newBlock
        self.startBlock = self.frame.startBlock
        self.nextBlock = self.frame.nextBlock
        self.setDocstring = self.frame.setDocstring
        # Delegate methods to graph object
        self.emit = self.graph.emit
        self.newBlock = self.graph.newBlock
        self.startBlock = self.graph.startBlock
        self.nextBlock = self.graph.nextBlock

        # Set flags based on future features
        for feature in ctx.futures:
@@ -269,7 +268,7 @@ def visitFunction(self, node):
        # MODULE gets the docstring of the last function?  But this changes the
        # output.
        if node.doc:
            self.setDocstring(node.doc)
            self.frame.setDocstring(node.doc)
        self.storeName(node.name)

    def visitLambda(self, node):
@@ -1276,7 +1275,7 @@ def _Start(self):
            self.frame.setDocstring(self.func.doc)

    def Finish(self):
        self.frame.startExitBlock()
        self.graph.startExitBlock()
        self.emit('LOAD_CONST', None)
        self.emit('RETURN_VALUE')

@@ -1288,7 +1287,7 @@ def _Start(self):
            self.frame.setFlag(CO_GENERATOR)

    def Finish(self):
        self.frame.startExitBlock()
        self.graph.startExitBlock()
        self.emit('RETURN_VALUE')


@@ -1298,7 +1297,7 @@ def _Start(self):
        self.frame.setFlag(CO_GENERATOR)  # It's always a generator

    def Finish(self):
        self.frame.startExitBlock()
        self.graph.startExitBlock()
        self.emit('RETURN_VALUE')


@@ -1326,10 +1325,10 @@ def FindLocals(self):

        self.frame.setFlag(CO_NEWLOCALS)
        if self.klass.doc:
            self.setDocstring(self.klass.doc)
            self.frame.setDocstring(self.klass.doc)

    def Finish(self):
        self.frame.startExitBlock()
        self.graph.startExitBlock()
        self.emit('LOAD_LOCALS')
        self.emit('RETURN_VALUE')

opy/compiler2/skeleton.py

@@ -97,7 +97,7 @@ def RunCompiler(f, filename, gr, start_symbol, mode):
  gen.Finish()

  # NOTE: This method has a pretty long pipeline too.
  co = frame.MakeCodeObject()
  co = pyassem.MakeCodeObject(gen.frame, gen.graph)
  return co

opy/opy_main.py

@@ -25,7 +25,6 @@

from .compiler2 import dis_tool
from .compiler2 import misc
from .compiler2 import pycodegen
from .compiler2 import skeleton
from .compiler2 import transformer

@@ -201,7 +200,7 @@ def OpyCommandMain(argv):
    py_path = argv[1]
    with open(py_path) as f:
      tokens = tokenize.generate_tokens(f.readline)
      p = parse.Parser(gr, convert=py2st)
      p = parse.Parser(gr, convert=skeleton.py2st)
      parse_tree = driver.PushTokens(p, tokens, gr.symbol2number['file_input'])

    if isinstance(parse_tree, tuple):
@@ -286,16 +285,6 @@ def OpyCommandMain(argv):
          h.update(b)
      print('%6d %s %s' % (os.path.getsize(path), h.hexdigest(), path))

  # TODO: Not used
  elif action == 'compile2':
    in_path = argv[1]
    out_path = argv[2]

    from compiler2 import pycodegen as pycodegen2
    from misc import stdlib_compile

    stdlib_compile.compileAndWrite(in_path, out_path, pycodegen2.compile)

  elif action == 'run':
    # TODO: Add an option like -v in __main__