Refactoring Compile(), pycodegen, pyassem.

- Do recursive code generation in the CodeGeneration rather than the
  ArgEncoder.  This is more logical.
- Make constant style consistent.
- Change mutable class variable __with_counter to be a member.
- Use itertools.count() instead of a global counter.

opy/compiler2/pyassem.py

@@ -78,14 +78,14 @@ def find_next():
    return order


def FlattenGraph(blocks):
def FlattenBlocks(blocks):
    insts = []

    pc = 0
    offsets = {}  # block -> bytecode offset
    for b in blocks:
        offsets[b] = pc
        for inst in b.getInstructions():
        for inst in b.Instructions():
            insts.append(inst)
            if len(inst) == 1:
                pc += 1
@@ -152,7 +152,7 @@ def emit(self, inst):
        op = inst[0]
        self.insts.append(inst)

    def getInstructions(self):
    def Instructions(self):
        return self.insts

    def addOutEdge(self, block):
@@ -393,10 +393,6 @@ def Run(self, insts):
    # TODO: This should just be a simple switch

    def _convert_LOAD_CONST(self, arg):
        from . import pycodegen
        from . import skeleton
        if isinstance(arg, pycodegen.CodeGenerator):
            arg = skeleton.MakeCodeObject(arg.frame, arg.graph)
        return _NameToIndex(arg, self.consts)

    def _convert_LOAD_FAST(self, arg):
@@ -649,13 +645,15 @@ def DUP_TOPX(self, argc):
        return argc


class GraphStackDepth(object):
    """Walk the CFG, computing the maximum stack depth.
class _GraphStackDepth(object):
    """Walk the CFG, computing the maximum stack depth."""

    'depths' is the stack effect of each basic block.  Then find the path
    through the code with the largest total effect.
    """
    def __init__(self, depths, exit_block):
        """
        Args:
           depths: is the stack effect of each basic block.  Then find the path
           through the code with the largest total effect.
        """
        self.depths = depths
        self.exit_block = exit_block
        self.seen = set()
@@ -675,3 +673,10 @@ def Max(self, block, d):
            return d

        return self.Max(self.exit_block, d)


def MaxStackDepth(block_depths, entry_block, exit_block):
    """Compute maximum stack depth for any path through the CFG."""
    g = _GraphStackDepth(block_depths, exit_block)
    return g.Max(entry_block, 0)

opy/compiler2/pycodegen.py

@@ -1,3 +1,5 @@
import itertools

from . import ast, pyassem, misc
from .visitor import ASTVisitor
from .consts import (
@@ -9,22 +11,38 @@
    CO_GENERATOR, CO_FUTURE_DIVISION,
    CO_FUTURE_ABSIMPORT, CO_FUTURE_WITH_STATEMENT, CO_FUTURE_PRINT_FUNCTION)

callfunc_opcode_info = {
_CALLFUNC_OPCODE_INFO = {
    # (Have *args, Have **args) : opcode
    (0,0) : "CALL_FUNCTION",
    (1,0) : "CALL_FUNCTION_VAR",
    (0,1) : "CALL_FUNCTION_KW",
    (1,1) : "CALL_FUNCTION_VAR_KW",
}

_AUGMENTED_OPCODE = {
    '+=' : 'INPLACE_ADD',
    '-=' : 'INPLACE_SUBTRACT',
    '*=' : 'INPLACE_MULTIPLY',
    '/=' : 'INPLACE_DIVIDE',
    '//=': 'INPLACE_FLOOR_DIVIDE',
    '%=' : 'INPLACE_MODULO',
    '**=': 'INPLACE_POWER',
    '>>=': 'INPLACE_RSHIFT',
    '<<=': 'INPLACE_LSHIFT',
    '&=' : 'INPLACE_AND',
    '^=' : 'INPLACE_XOR',
    '|=' : 'INPLACE_OR',
}


LOOP = 1
EXCEPT = 2
TRY_FINALLY = 3
END_FINALLY = 4


# TODO: Move this mutable global somewhere.
gLambdaCounter = 0
# TODO: Move this to _ModuleContext so it's not a mutable global?
gLambdaCounter = itertools.count()


class LocalNameFinder(ASTVisitor):
@@ -72,10 +90,7 @@ def visitAssName(self, node):


def is_constant_false(node):
    if isinstance(node, ast.Const):
        if not node.value:
            return 1
    return 0
    return isinstance(node, ast.Const) and not node.value


class Stack(list):
@@ -105,6 +120,7 @@ def __init__(self, ctx, frame, graph):

        self.locals = Stack()
        self.setups = Stack()
        self.__with_count = 0
        self.last_lineno = None
        self._div_op = "BINARY_DIVIDE"

@@ -273,9 +289,7 @@ def visitFunction(self, node):
        self.storeName(node.name)

    def visitLambda(self, node):
        global gLambdaCounter
        obj_name = "<lambda.%d>" % gLambdaCounter
        gLambdaCounter += 1
        obj_name = "<lambda.%d>" % gLambdaCounter.next()

        _CheckNoTupleArgs(node)
        frame = pyassem.Frame(obj_name, self.ctx.filename, optimized=1)
@@ -574,22 +588,25 @@ def _makeClosure(self, gen, args):
        afterward, so I guess we can't do that here?
        """
        frees = gen.scope.get_free_vars()

        # Recursive call!
        from . import skeleton
        co = skeleton.MakeCodeObject(gen.frame, gen.graph)

        if frees:
            for name in frees:
                self.emit('LOAD_CLOSURE', name)
            self.emit('BUILD_TUPLE', len(frees))
            self.emit('LOAD_CONST', gen)
            self.emit('LOAD_CONST', co)
            self.emit('MAKE_CLOSURE', args)
        else:
            self.emit('LOAD_CONST', gen)
            self.emit('LOAD_CONST', co)
            self.emit('MAKE_FUNCTION', args)

    def visitGenExpr(self, node):
        isLambda = 1  # TODO: Shouldn't be a lambda?  Note Finish().
        if isLambda:
            global gLambdaCounter
            obj_name = "<lambda.%d>" % gLambdaCounter
            gLambdaCounter += 1
            obj_name = "<lambda.%d>" % gLambdaCounter.next()
        else:
            # TODO: enable this.  This is more like CPython.  Note that I worked
            # worked around a bug in byterun due to NOT having this.
@@ -772,8 +789,6 @@ def visitTryFinally(self, node):
        self.emit('END_FINALLY')
        self.setups.pop()

    __with_count = 0

    def visitWith(self, node):
        body = self.newBlock()
        final = self.newBlock()
@@ -930,24 +945,9 @@ def visitAugAssign(self, node):
        aug_node = wrap_aug(node.node)
        self.visit(aug_node, "load")
        self.visit(node.expr)
        self.emit(self._augmented_opcode[node.op])
        self.emit(_AUGMENTED_OPCODE[node.op])
        self.visit(aug_node, "store")

    _augmented_opcode = {
        '+=' : 'INPLACE_ADD',
        '-=' : 'INPLACE_SUBTRACT',
        '*=' : 'INPLACE_MULTIPLY',
        '/=' : 'INPLACE_DIVIDE',
        '//=': 'INPLACE_FLOOR_DIVIDE',
        '%=' : 'INPLACE_MODULO',
        '**=': 'INPLACE_POWER',
        '>>=': 'INPLACE_RSHIFT',
        '<<=': 'INPLACE_LSHIFT',
        '&=' : 'INPLACE_AND',
        '^=' : 'INPLACE_XOR',
        '|=' : 'INPLACE_OR',
        }

    def visitAugName(self, node, mode):
        if mode == "load":
            self.loadName(node.name)
@@ -1016,7 +1016,7 @@ def visitCallFunc(self, node):
            self.visit(node.dstar_args)
        have_star = node.star_args is not None
        have_dstar = node.dstar_args is not None
        opcode = callfunc_opcode_info[have_star, have_dstar]
        opcode = _CALLFUNC_OPCODE_INFO[have_star, have_dstar]
        self.emit(opcode, kw << 8 | pos)

    def visitPrint(self, node, newline=0):
@@ -1347,6 +1347,7 @@ def visitAssName(self, node):
            self.op = node.flags
        elif self.op != node.flags:
            raise ValueError, "mixed ops in stmt"

    visitAssAttr = visitAssName
    visitSubscript = visitAssName

@@ -1370,6 +1371,7 @@ def __init__(self, obj):
    def __getattr__(self, attr):
        return getattr(self.obj, attr)


class AugGetattr(Delegator):
    pass

opy/compiler2/skeleton.py

@@ -49,20 +49,24 @@ def MakeCodeObject(frame, graph):
    Called by Compile below, and also recursively by ArgEncoder.
    """
    # Compute stack depth per basic block.
    depths = {}
    b = pyassem.BlockStackDepth()
    for block in graph.blocks:
        depths[block] = b.Sum(block.getInstructions())
    block_depths = {
        block: b.Sum(block.Instructions()) for block in graph.blocks
    }

    # Compute maximum stack depth for any path through the CFG.
    g = pyassem.GraphStackDepth(depths, graph.exit)
    stacksize = g.Max(graph.entry, 0)
    stacksize = pyassem.MaxStackDepth(block_depths, graph.entry, graph.exit)

    # Order blocks so jump offsets can be encoded.
    blocks = pyassem.OrderBlocks(graph.entry, graph.exit)
    insts, block_offsets = pyassem.FlattenGraph(blocks)

    # Produce a stream of initial instructions.
    insts, block_offsets = pyassem.FlattenBlocks(blocks)

    # Now that we know the offsets, make another pass.
    pyassem.PatchJumps(insts, block_offsets)

    # What variables should be available at runtime?

    cellvars = pyassem.ReorderCellVars(frame)
    consts = [frame.docstring]
    names = []
@@ -74,9 +78,10 @@ def MakeCodeObject(frame, graph):
    # Convert arguments from symbolic to concrete form.
    enc = pyassem.ArgEncoder(frame.klass, consts, names, frame.varnames,
                             closure)
    # Mutates not only insts, but also consts, names, etc.
    # Mutates not only insts, but also appends to consts, names, etc.
    enc.Run(insts)

    # Binary encoding.
    a = pyassem.Assembler()
    bytecode, firstline, lnotab = a.Run(insts)

@@ -150,4 +155,8 @@ def Compile(f, filename, gr, start_symbol, mode):
  gen.Dispatch(as_tree)  # mutates graph
  gen.Finish()

  return MakeCodeObject(frame, graph)
  co = MakeCodeObject(frame, graph)

  # TODO: Could call marshal.dump here?
  return co