Move skeleton.py to opy/ root, since it uses both pgen2 and compiler2.

Move MakeCodeObject() back to pyassem, since it's called recursively.

opy/compiler2/pyassem.py

@@ -3,6 +3,7 @@

import dis
import itertools
import types

from .consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS

@@ -677,3 +678,56 @@ def MaxStackDepth(block_depths, entry_block, exit_block):
    g = _GraphStackDepth(block_depths, exit_block)
    return g.Max(entry_block, 0)


def MakeCodeObject(frame, graph):
    """Order blocks, encode instructions, and create types.CodeType().

    Called by Compile below, and also recursively by ArgEncoder.
    """
    # Compute stack depth per basic block.
    b = BlockStackDepth()
    block_depths = {
        block: b.Sum(block.Instructions()) for block in graph.blocks
    }

    stacksize = MaxStackDepth(block_depths, graph.entry, graph.exit)

    # Order blocks so jump offsets can be encoded.
    blocks = OrderBlocks(graph.entry, graph.exit)

    # Produce a stream of initial instructions.
    insts, block_offsets = FlattenBlocks(blocks)

    # Now that we know the offsets, make another pass.
    PatchJumps(insts, block_offsets)

    # What variables should be available at runtime?

    cellvars = ReorderCellVars(frame)
    consts = [frame.docstring]
    names = []
    # The closure list is used to track the order of cell variables and
    # free variables in the resulting code object.  The offsets used by
    # LOAD_CLOSURE/LOAD_DEREF refer to both kinds of variables.
    closure = cellvars + frame.freevars

    # Convert arguments from symbolic to concrete form.
    enc = ArgEncoder(frame.klass, consts, names, frame.varnames,
                             closure)
    # Mutates not only insts, but also appends to consts, names, etc.
    enc.Run(insts)

    # Binary encoding.
    a = Assembler()
    bytecode, firstline, lnotab = a.Run(insts)

    return types.CodeType(
        frame.ArgCount(), frame.NumLocals(), stacksize, frame.flags,
        bytecode,
        tuple(consts),
        tuple(names),
        tuple(frame.varnames),
        frame.filename, frame.name, firstline,
        lnotab,
        tuple(frame.freevars),
        tuple(cellvars))

opy/compiler2/pycodegen.py

@@ -590,8 +590,7 @@ def _makeClosure(self, gen, args):
        frees = gen.scope.get_free_vars()

        # Recursive call!
        from . import skeleton
        co = skeleton.MakeCodeObject(gen.frame, gen.graph)
        co = pyassem.MakeCodeObject(gen.frame, gen.graph)

        if frees:
            for name in frees:

opy/opy_main.py

@@ -18,14 +18,15 @@
#this_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
#sys.path.append(os.path.join(this_dir))

from . import pytree
from . import skeleton

from .pgen2 import driver, parse, pgen, grammar
from .pgen2 import token
from .pgen2 import tokenize
from . import pytree

from .compiler2 import dis_tool
from .compiler2 import misc
from .compiler2 import skeleton
from .compiler2 import transformer

# Disabled for now because byterun imports 'six', and that breaks the build.

opy/compiler2/skeleton.py → opy/skeleton.py

@@ -3,18 +3,16 @@
skeleton.py: The compiler pipeline.
"""

import types
from .pgen2 import tokenize
from .pgen2 import driver
from .pgen2 import parse

from ..pgen2 import tokenize
from ..pgen2 import driver
from ..pgen2 import parse

from . import future
from . import pyassem
from . import pycodegen
from . import syntax
from . import symbols
from . import transformer
from .compiler2 import future
from .compiler2 import pyassem
from .compiler2 import pycodegen
from .compiler2 import syntax
from .compiler2 import symbols
from .compiler2 import transformer


class _ModuleContext(object):
@@ -43,60 +41,6 @@ def py2st(unused_gr, raw_node):
    return (typ, value, lineno, column)


def MakeCodeObject(frame, graph):
    """Order blocks, encode instructions, and create types.CodeType().

    Called by Compile below, and also recursively by ArgEncoder.
    """
    # Compute stack depth per basic block.
    b = pyassem.BlockStackDepth()
    block_depths = {
        block: b.Sum(block.Instructions()) for block in graph.blocks
    }

    stacksize = pyassem.MaxStackDepth(block_depths, graph.entry, graph.exit)

    # Order blocks so jump offsets can be encoded.
    blocks = pyassem.OrderBlocks(graph.entry, graph.exit)

    # Produce a stream of initial instructions.
    insts, block_offsets = pyassem.FlattenBlocks(blocks)

    # Now that we know the offsets, make another pass.
    pyassem.PatchJumps(insts, block_offsets)

    # What variables should be available at runtime?

    cellvars = pyassem.ReorderCellVars(frame)
    consts = [frame.docstring]
    names = []
    # The closure list is used to track the order of cell variables and
    # free variables in the resulting code object.  The offsets used by
    # LOAD_CLOSURE/LOAD_DEREF refer to both kinds of variables.
    closure = cellvars + frame.freevars

    # Convert arguments from symbolic to concrete form.
    enc = pyassem.ArgEncoder(frame.klass, consts, names, frame.varnames,
                             closure)
    # Mutates not only insts, but also appends to consts, names, etc.
    enc.Run(insts)

    # Binary encoding.
    a = pyassem.Assembler()
    bytecode, firstline, lnotab = a.Run(insts)

    return types.CodeType(
        frame.ArgCount(), frame.NumLocals(), stacksize, frame.flags,
        bytecode,
        tuple(consts),
        tuple(names),
        tuple(frame.varnames),
        frame.filename, frame.name, firstline,
        lnotab,
        tuple(frame.freevars),
        tuple(cellvars))


def Compile(f, filename, gr, start_symbol, mode):
  """Run the full compiler pipeline.

@@ -155,8 +99,7 @@ def Compile(f, filename, gr, start_symbol, mode):
  gen.Dispatch(as_tree)  # mutates graph
  gen.Finish()

  co = MakeCodeObject(frame, graph)
  co = pyassem.MakeCodeObject(frame, graph)

  # TODO: Could call marshal.dump here?
  return co