Start separating the concept of FlowGraph and Frame.

opy/byterun/pyobj.pyc changed, because it has a class called Frame!

Rename StackDepthTracker -> BlockStackDepth.

opy/compiler2/pyassem.py

@@ -205,7 +205,7 @@ def getContainedGraphs(self):
        return contained


class FlowGraph(object):
class _FlowGraph(object):

    def __init__(self):
        self.current = self.entry = Block()
@@ -273,8 +273,13 @@ def getContainedGraphs(self):
        return l


# Placeholder
class FlowGraph(object):
  pass


# TODO: Unify FlowGraph and PyFlowGraph.
class PyFlowGraph(FlowGraph):
class Frame(_FlowGraph):
    """Something that gets turned into a single code object.

    Code objects and consts are mutually recursive.
@@ -285,7 +290,7 @@ def __init__(self, name, filename, optimized=0, klass=None):
        Args:
          klass: Whether we're compiling a class block.
        """
        FlowGraph.__init__(self)
        _FlowGraph.__init__(self)
        self.name = name  # name that is put in the code object
        self.filename = filename
        self.flags = (CO_OPTIMIZED | CO_NEWLOCALS) if optimized else 0
@@ -337,12 +342,14 @@ def _ReorderCellVars(self):
    def MakeCodeObject(self):
        """Order blocks, encode instructions, and create types.CodeType()."""

        # Compute stack depth per basic block.
        depths = {}
        for b in self.blocks:
            depths[b] = TRACKER.findDepth(b.getInstructions())
            depths[b] = BLOCK_STACK_DEPTH.Sum(b.getInstructions())

        b = BlockStackDepth(depths, self.exit)
        stacksize = b.Max(self.entry, 0)
        # Compute maximum stack depth for any control flow.
        g = GraphStackDepth(depths, self.exit)
        stacksize = g.Max(self.entry, 0)

        blocks = OrderBlocks(self.entry, self.exit)
        insts = FlattenGraph(blocks)
@@ -432,7 +439,8 @@ def Run(self, insts):
    def _convert_LOAD_CONST(self, arg):
        from . import pycodegen
        if isinstance(arg, pycodegen.CodeGenerator):
            arg = arg.graph.MakeCodeObject()
            arg = arg.frame.MakeCodeObject()
            # arg.code
        return _NameToIndex(arg, self.consts)

    def _convert_LOAD_FAST(self, arg):
@@ -568,11 +576,11 @@ def Run(self, insts):
        return bytecode, self.firstline, lnotab


class StackDepthTracker(object):
class BlockStackDepth(object):
    # XXX 1. need to keep track of stack depth on jumps
    # XXX 2. at least partly as a result, this code is broken

    def findDepth(self, insts, debug=0):
    def Sum(self, insts, debug=0):
        depth = 0
        maxDepth = 0

@@ -685,17 +693,15 @@ def DUP_TOPX(self, argc):
        return argc


TRACKER = StackDepthTracker()
BLOCK_STACK_DEPTH = BlockStackDepth()


class BlockStackDepth(object):
class GraphStackDepth(object):
    """Walk the CFG, computing the maximum stack depth.

    Approach is to compute the stack effect of each basic block.
    Then find the path through the code with the largest total
    effect.
    'depths' is the stack effect of each basic block.  Then find the path
    through the code with the largest total effect.
    """

    def __init__(self, depths, exit_block):
        self.depths = depths
        self.exit_block = exit_block
@@ -705,9 +711,10 @@ def Max(self, block, d):
        if block in self.seen:
            return d
        self.seen.add(block)

        d += self.depths[block]
        children = block.get_children()

        children = block.get_children()
        if children:
            return max(self.Max(c, d) for c in children)

opy/compiler2/pycodegen.py

@@ -88,17 +88,14 @@ def top(self):


class CodeGenerator(ASTVisitor):
    """Defines basic code generator for Python bytecode
    """Abstract class for modules, classes, functions."""

    This class is an abstract base class.  Concrete subclasses must
    define an __init__() that defines self.graph and then calls the
    __init__() defined in this class.
    """
    optimized = 0  # is namespace access optimized?
    class_name = None  # provide default for instance variable

    def __init__(self, graph, ctx):
    def __init__(self, frame, graph, ctx):
        ASTVisitor.__init__(self)
        self.frame = frame
        self.graph = graph
        self.ctx = ctx  # passed down to child CodeGenerator instances

@@ -112,23 +109,23 @@ def __init__(self, graph, ctx):
        self._div_op = "BINARY_DIVIDE"

        # Set up methods
        self.emit = self.graph.emit
        self.newBlock = self.graph.newBlock
        self.startBlock = self.graph.startBlock
        self.nextBlock = self.graph.nextBlock
        self.setDocstring = self.graph.setDocstring
        self.emit = self.frame.emit
        self.newBlock = self.frame.newBlock
        self.startBlock = self.frame.startBlock
        self.nextBlock = self.frame.nextBlock
        self.setDocstring = self.frame.setDocstring

        # Set flags based on future features
        for feature in ctx.futures:
            if feature == "division":
                self.graph.setFlag(CO_FUTURE_DIVISION)
                self.frame.setFlag(CO_FUTURE_DIVISION)
                self._div_op = "BINARY_TRUE_DIVIDE"
            elif feature == "absolute_import":
                self.graph.setFlag(CO_FUTURE_ABSIMPORT)
                self.frame.setFlag(CO_FUTURE_ABSIMPORT)
            elif feature == "with_statement":
                self.graph.setFlag(CO_FUTURE_WITH_STATEMENT)
                self.frame.setFlag(CO_FUTURE_WITH_STATEMENT)
            elif feature == "print_function":
                self.graph.setFlag(CO_FUTURE_PRINT_FUNCTION)
                self.frame.setFlag(CO_FUTURE_PRINT_FUNCTION)

    #
    # Two methods that subclasses must implement.
@@ -141,7 +138,7 @@ def Finish(self):
        raise NotImplementedError

    def Start(self):
        self.graph.setVars(self.scope.get_free_vars(),
        self.frame.setVars(self.scope.get_free_vars(),
                           self.scope.get_cell_vars())
        self._Start()

@@ -258,10 +255,12 @@ def visitFunction(self, node):
            ndecorators = 0

        _CheckNoTupleArgs(node)
        graph = pyassem.PyFlowGraph(node.name, self.ctx.filename, optimized=1)
        graph.setArgs(node.argnames)

        gen = FunctionCodeGenerator(graph, self.ctx, node, self.class_name)
        frame = pyassem.Frame(node.name, self.ctx.filename, optimized=1)
        frame.setArgs(node.argnames)
        graph = pyassem.FlowGraph()

        gen = FunctionCodeGenerator(frame, graph, self.ctx, node, self.class_name)

        self._funcOrLambda(node, gen, ndecorators)

@@ -279,10 +278,11 @@ def visitLambda(self, node):
        gLambdaCounter += 1

        _CheckNoTupleArgs(node)
        graph = pyassem.PyFlowGraph(obj_name, self.ctx.filename, optimized=1)
        graph.setArgs(node.argnames)
        frame = pyassem.Frame(obj_name, self.ctx.filename, optimized=1)
        frame.setArgs(node.argnames)
        graph = pyassem.FlowGraph()

        gen = LambdaCodeGenerator(graph, self.ctx, node, self.class_name)
        gen = LambdaCodeGenerator(frame, graph, self.ctx, node, self.class_name)

        self._funcOrLambda(node, gen, 0)

@@ -300,9 +300,9 @@ def _funcOrLambda(self, node, gen, ndecorators):
            self.emit('CALL_FUNCTION', 1)

    def visitClass(self, node):
        graph = pyassem.PyFlowGraph(node.name, self.ctx.filename,
                                    optimized=0, klass=1)
        gen = ClassCodeGenerator(graph, self.ctx, node)
        frame = pyassem.Frame(node.name, self.ctx.filename, optimized=0, klass=1)
        graph = pyassem.FlowGraph()
        gen = ClassCodeGenerator(frame, graph, self.ctx, node)

        gen.Start()
        gen.FindLocals()
@@ -567,6 +567,11 @@ def visitListCompIf(self, node, branch):
        self.newBlock()

    def _makeClosure(self, gen, args):
        """Emit LOAD_CONST of this generator.

        ArgEncoder calls MakeCodeObject on it.  There are a few instructions
        afterward, so I guess we can't do that here?
        """
        frees = gen.scope.get_free_vars()
        if frees:
            for name in frees:
@@ -591,9 +596,11 @@ def visitGenExpr(self, node):
            # That workaround may no longer be necessary if we switch.
            obj_name = '<genexpr>'

        graph = pyassem.PyFlowGraph(obj_name, self.ctx.filename, optimized=1)
        graph.setArgs(node.argnames)
        gen = GenExprCodeGenerator(graph, self.ctx, node, self.class_name)
        frame = pyassem.Frame(obj_name, self.ctx.filename, optimized=1)
        frame.setArgs(node.argnames)
        graph = pyassem.FlowGraph()
        gen = GenExprCodeGenerator(frame, graph, self.ctx, node,
                                   self.class_name)

        gen.Start()
        gen.FindLocals()
@@ -827,7 +834,7 @@ def visitPass(self, node):

    def visitImport(self, node):
        self.set_lineno(node)
        level = 0 if self.graph.checkFlag(CO_FUTURE_ABSIMPORT) else -1
        level = 0 if self.frame.checkFlag(CO_FUTURE_ABSIMPORT) else -1
        for name, alias in node.names:
            self.emit('LOAD_CONST', level)
            self.emit('LOAD_CONST', None)
@@ -842,7 +849,7 @@ def visitImport(self, node):
    def visitFrom(self, node):
        self.set_lineno(node)
        level = node.level
        if level == 0 and not self.graph.checkFlag(CO_FUTURE_ABSIMPORT):
        if level == 0 and not self.frame.checkFlag(CO_FUTURE_ABSIMPORT):
            level = -1
        fromlist = tuple(name for (name, alias) in node.names)
        self.emit('LOAD_CONST', level)
@@ -1239,8 +1246,8 @@ class _FunctionCodeGenerator(CodeGenerator):
    """Abstract class."""
    optimized = 1

    def __init__(self, graph, ctx, func, class_name):
        CodeGenerator.__init__(self, graph, ctx)
    def __init__(self, frame, graph, ctx, func, class_name):
        CodeGenerator.__init__(self, frame, graph, ctx)
        self.func = func
        self.class_name = class_name

@@ -1254,22 +1261,22 @@ def FindLocals(self):
        self.locals.push(lnf.getLocals())

        if func.varargs:
            self.graph.setFlag(CO_VARARGS)
            self.frame.setFlag(CO_VARARGS)
        if func.kwargs:
            self.graph.setFlag(CO_VARKEYWORDS)
            self.frame.setFlag(CO_VARKEYWORDS)
        self.set_lineno(func)


class FunctionCodeGenerator(_FunctionCodeGenerator):

    def _Start(self):
        if self.scope.generator is not None:  # does it have yield in it?
            self.graph.setFlag(CO_GENERATOR)
            self.frame.setFlag(CO_GENERATOR)
        if self.func.doc:
            self.setDocstring(self.func.doc)
            self.frame.setDocstring(self.func.doc)

    def Finish(self):
        self.graph.startExitBlock()
        self.frame.startExitBlock()
        self.emit('LOAD_CONST', None)
        self.emit('RETURN_VALUE')

@@ -1278,27 +1285,27 @@ class LambdaCodeGenerator(_FunctionCodeGenerator):

    def _Start(self):
        if self.scope.generator is not None:  # does it have yield in it?
            self.graph.setFlag(CO_GENERATOR)
            self.frame.setFlag(CO_GENERATOR)

    def Finish(self):
        self.graph.startExitBlock()
        self.frame.startExitBlock()
        self.emit('RETURN_VALUE')


class GenExprCodeGenerator(_FunctionCodeGenerator):

    def _Start(self):
        self.graph.setFlag(CO_GENERATOR)  # It's always a generator
        self.frame.setFlag(CO_GENERATOR)  # It's always a generator

    def Finish(self):
        self.graph.startExitBlock()
        self.frame.startExitBlock()
        self.emit('RETURN_VALUE')


class ClassCodeGenerator(CodeGenerator):

    def __init__(self, graph, ctx, klass):
        CodeGenerator.__init__(self, graph, ctx)
    def __init__(self, frame, graph, ctx, klass):
        CodeGenerator.__init__(self, frame, graph, ctx)
        self.klass = klass

        self.class_name = klass.name
@@ -1317,12 +1324,12 @@ def FindLocals(self):
        lnf.Dispatch(self.klass.code)
        self.locals.push(lnf.getLocals())

        self.graph.setFlag(CO_NEWLOCALS)
        self.frame.setFlag(CO_NEWLOCALS)
        if self.klass.doc:
            self.setDocstring(self.klass.doc)

    def Finish(self):
        self.graph.startExitBlock()
        self.frame.startExitBlock()
        self.emit('LOAD_LOCALS')
        self.emit('RETURN_VALUE')

opy/opy_main.py

@@ -177,15 +177,16 @@ def RunCompiler(f, filename, gr, start_symbol, mode):
  s = symbols.SymbolVisitor()
  s.Dispatch(as_tree)

  graph = pyassem.FlowGraph()
  if mode == "single":
      # NOTE: the name of the flow graph is a comment, not exposed to users.
      graph = pyassem.PyFlowGraph("<interactive>", filename)
      frame = pyassem.Frame("<interactive>", filename)
      ctx = _ModuleContext(filename, s.scopes)
      gen = pycodegen.InteractiveCodeGenerator(graph, ctx)
      gen = pycodegen.InteractiveCodeGenerator(frame, graph, ctx)
      gen.set_lineno(as_tree)

  elif mode == "exec":
      graph = pyassem.PyFlowGraph("<module>", filename)
      frame = pyassem.Frame("<module>", filename)

      # TODO: Does this need to be made more efficient?
      p1 = future.FutureParser()
@@ -194,12 +195,12 @@ def RunCompiler(f, filename, gr, start_symbol, mode):
      p2.Dispatch(as_tree)

      ctx = _ModuleContext(filename, s.scopes, futures=p1.get_features())
      gen = pycodegen.TopLevelCodeGenerator(graph, ctx)
      gen = pycodegen.TopLevelCodeGenerator(frame, graph, ctx)

  elif mode == "eval":
      graph = pyassem.PyFlowGraph("<expression>", filename)
      frame = pyassem.Frame("<expression>", filename)
      ctx = _ModuleContext(filename, s.scopes)
      gen = pycodegen.TopLevelCodeGenerator(graph, ctx)
      gen = pycodegen.TopLevelCodeGenerator(frame, graph, ctx)

  else:
      raise ValueError("compile() 3rd arg must be 'exec' or "
@@ -210,7 +211,7 @@ def RunCompiler(f, filename, gr, start_symbol, mode):
  gen.Finish()

  # NOTE: This method has a pretty long pipeline too.
  co = graph.MakeCodeObject()
  co = frame.MakeCodeObject()
  return co