Move MakeCodeObject() into skeleton.py.

pycodegen style: put the constant ctx argument first, before the mutable frame and graph arguments. Update golden checksums.
oilshell · Mar 20, 2018 · fed6d7e24672ec281216bf3ef89c9da4167ebef9 · fed6d7e
1 parent 19b9876
commit fed6d7e24672ec281216bf3ef89c9da4167ebef9
Unified Split

Showing with 96 additions and 95 deletions.

+3 −3 opy/_regtest/dis-md5.golden.txt

+14 −61 opy/compiler2/pyassem.py

+12 −10 opy/compiler2/pycodegen.py

+67 −21 opy/compiler2/skeleton.py
diff --git a/opy/_regtest/dis-md5.golden.txt b/opy/_regtest/dis-md5.golden.txt
@@ -150,11 +150,11 @@
  9249 ff523d9b89278d6ed3f57247ea1f8fe7 _tmp/regtest/asdl/py_meta.pyc
  9316 64bd9b485abd8deafd992c1558b89ca9 _tmp/regtest/asdl/tdop.pyc
  9430 55c117fa40d8b65b5caf648db19bab2c _tmp/regtest/osh/lex_test.pyc
-  9568 983c982a13d92b8ea80e3e0544bcc4bb _tmp/regtest/opy/byterun/pyobj.pyc
+  9573 f4b5773c8ba0471ba81467e99b12ae47 _tmp/regtest/opy/byterun/pyobj.pyc
  9874 6a7451da64ece36368705f98005046ac _tmp/regtest/core/legacy.pyc
  9926 235ebfb8f8a106983664f358ee4579cb _tmp/regtest/core/util.pyc
 10513 54a627997dabacb3c7c668b3b493d22e _tmp/regtest/runpy.pyc
- 10574 844c3e8279a3c7be9ef97e014ec1e99b _tmp/regtest/opy/opy_main.pyc
+ 10560 5f30d8c99b518534251459804d8e6af2 _tmp/regtest/opy/opy_main.pyc
 10802 41b4b7f7c82b5975c2b78f9726befd1e _tmp/regtest/UserDict.pyc
 11111 516ad8bd7dddeab5dcfe1cca2bbf2b94 _tmp/regtest/core/id_kind.pyc
 11118 c012a0c4650a7b80d6b3c51962a3221c _tmp/regtest/_weakrefset.pyc
@@ -202,7 +202,7 @@
 28978 581863c0d28c4f23d0eaaaab3a525f33 _tmp/regtest/test/sh_spec.pyc
 29159 58e8951b48374c0368253e32c9550075 _tmp/regtest/_abcoll.pyc
 29440 5ae214959a39478444778fa9800f4dd6 _tmp/regtest/collections.pyc
- 30243 9a54545f2f70fc1b265676e07f4f258c _tmp/regtest/opy/compiler2/pyassem.pyc
+ 30204 7b90326a7eab1a0f0d42f21382c966e3 _tmp/regtest/opy/compiler2/pyassem.pyc
 30811 02d4a38a06b87bc1875e62ffe18bb88a _tmp/regtest/core/builtin.pyc
 32295 29b6c155971a78b46cae1c83500f7efb _tmp/regtest/osh/word_parse.pyc
 32295 3f24a5980d6761b920977cfa7856a65d _tmp/regtest/opy/pytree.pyc
diff --git a/opy/compiler2/pyassem.py b/opy/compiler2/pyassem.py
@@ -2,7 +2,6 @@
 from __future__ import print_function

 import dis
-import types

 from .consts import CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS

@@ -320,8 +319,19 @@ def setFlag(self, flag):
    def checkFlag(self, flag):
        return bool(self.flags & flag)

+    def NumLocals(self):
+        return len(self.varnames) if self.flags & CO_NEWLOCALS else 0

-def _ReorderCellVars(frame):
+    def ArgCount(self):
+        argcount = self.argcount
+        if self.flags & CO_VARKEYWORDS:
+            argcount -= 1
+        if self.flags & CO_VARARGS:
+            argcount -= 1
+        return argcount
+
+
+def ReorderCellVars(frame):
    """Reorder cellvars so the ones in self.varnames are first.

    And prune from freevars (?)
@@ -334,60 +344,6 @@ def _ReorderCellVars(frame):
    return cellvars


-def MakeCodeObject(frame, graph):
-    """Order blocks, encode instructions, and create types.CodeType()."""
-
-    # Compute stack depth per basic block.
-    depths = {}
-    for b in graph.blocks:
-        depths[b] = BLOCK_STACK_DEPTH.Sum(b.getInstructions())
-
-    # Compute maximum stack depth for any control flow.
-    g = GraphStackDepth(depths, graph.exit)
-    stacksize = g.Max(graph.entry, 0)
-
-    blocks = OrderBlocks(graph.entry, graph.exit)
-    insts = FlattenGraph(blocks)
-
-    cellvars = _ReorderCellVars(frame)
-    consts = [frame.docstring]
-    names = []
-    # The closure list is used to track the order of cell variables and
-    # free variables in the resulting code object.  The offsets used by
-    # LOAD_CLOSURE/LOAD_DEREF refer to both kinds of variables.
-    closure = cellvars + frame.freevars
-
-    # Convert arguments from symbolic to concrete form.
-    enc = ArgEncoder(frame.klass, consts, names, frame.varnames, closure)
-    # Mutates not only insts, but also consts, names, etc.
-    enc.Run(insts)
-
-    if frame.flags & CO_NEWLOCALS:
-        nlocals = len(frame.varnames)
-    else:
-        nlocals = 0
-
-    argcount = frame.argcount
-    if frame.flags & CO_VARKEYWORDS:
-        argcount -= 1
-    if frame.flags & CO_VARARGS:
-        argcount -= 1
-
-    a = Assembler()
-    bytecode, firstline, lnotab = a.Run(insts)
-
-    return types.CodeType(
-        argcount, nlocals, stacksize, frame.flags,
-        bytecode,
-        tuple(consts),
-        tuple(names),
-        tuple(frame.varnames),
-        frame.filename, frame.name, firstline,
-        lnotab,
-        tuple(frame.freevars),
-        tuple(cellvars))
-
-
 def _NameToIndex(name, L):
    """Return index of name in list, appending if necessary

@@ -434,9 +390,9 @@ def Run(self, insts):

    def _convert_LOAD_CONST(self, arg):
        from . import pycodegen
+        from . import skeleton
        if isinstance(arg, pycodegen.CodeGenerator):
-            arg = MakeCodeObject(arg.frame, arg.graph)
-            # arg.code
+            arg = skeleton.MakeCodeObject(arg.frame, arg.graph)
        return _NameToIndex(arg, self.consts)

    def _convert_LOAD_FAST(self, arg):
@@ -689,9 +645,6 @@ def DUP_TOPX(self, argc):
        return argc


-BLOCK_STACK_DEPTH = BlockStackDepth()
-
-
 class GraphStackDepth(object):
    """Walk the CFG, computing the maximum stack depth.

diff --git a/opy/compiler2/pycodegen.py b/opy/compiler2/pycodegen.py
@@ -93,11 +93,11 @@ class CodeGenerator(ASTVisitor):
    optimized = 0  # is namespace access optimized?
    class_name = None  # provide default for instance variable

-    def __init__(self, frame, graph, ctx):
+    def __init__(self, ctx, frame, graph):
        ASTVisitor.__init__(self)
+        self.ctx = ctx  # passed down to child CodeGenerator instances
        self.frame = frame
        self.graph = graph
-        self.ctx = ctx  # passed down to child CodeGenerator instances

        # Set by visitModule, visitExpression (for eval), or by subclass
        # constructor.
@@ -259,7 +259,8 @@ def visitFunction(self, node):
        frame.setArgs(node.argnames)
        graph = pyassem.FlowGraph()

-        gen = FunctionCodeGenerator(frame, graph, self.ctx, node, self.class_name)
+        gen = FunctionCodeGenerator(self.ctx, frame, graph, node,
+                                    self.class_name)

        self._funcOrLambda(node, gen, ndecorators)

@@ -281,7 +282,8 @@ def visitLambda(self, node):
        frame.setArgs(node.argnames)
        graph = pyassem.FlowGraph()

-        gen = LambdaCodeGenerator(frame, graph, self.ctx, node, self.class_name)
+        gen = LambdaCodeGenerator(self.ctx, frame, graph, node,
+                                  self.class_name)

        self._funcOrLambda(node, gen, 0)

@@ -301,7 +303,7 @@ def _funcOrLambda(self, node, gen, ndecorators):
    def visitClass(self, node):
        frame = pyassem.Frame(node.name, self.ctx.filename, optimized=0, klass=1)
        graph = pyassem.FlowGraph()
-        gen = ClassCodeGenerator(frame, graph, self.ctx, node)
+        gen = ClassCodeGenerator(self.ctx, frame, graph, node)

        gen.Start()
        gen.FindLocals()
@@ -598,7 +600,7 @@ def visitGenExpr(self, node):
        frame = pyassem.Frame(obj_name, self.ctx.filename, optimized=1)
        frame.setArgs(node.argnames)
        graph = pyassem.FlowGraph()
-        gen = GenExprCodeGenerator(frame, graph, self.ctx, node,
+        gen = GenExprCodeGenerator(self.ctx, frame, graph, node,
                                   self.class_name)

        gen.Start()
@@ -1245,8 +1247,8 @@ class _FunctionCodeGenerator(CodeGenerator):
    """Abstract class."""
    optimized = 1

-    def __init__(self, frame, graph, ctx, func, class_name):
-        CodeGenerator.__init__(self, frame, graph, ctx)
+    def __init__(self, ctx, frame, graph, func, class_name):
+        CodeGenerator.__init__(self, ctx, frame, graph)
        self.func = func
        self.class_name = class_name

@@ -1303,8 +1305,8 @@ def Finish(self):

 class ClassCodeGenerator(CodeGenerator):

-    def __init__(self, frame, graph, ctx, klass):
-        CodeGenerator.__init__(self, frame, graph, ctx)
+    def __init__(self, ctx, frame, graph, klass):
+        CodeGenerator.__init__(self, ctx, frame, graph)
        self.klass = klass

        self.class_name = klass.name
diff --git a/opy/compiler2/skeleton.py b/opy/compiler2/skeleton.py
@@ -3,6 +3,8 @@
 skeleton.py: The compiler pipeline.
 """

+import types
+
 from ..pgen2 import tokenize
 from ..pgen2 import driver
 from ..pgen2 import parse
@@ -15,6 +17,15 @@
 from . import transformer


+class _ModuleContext(object):
+  """Module-level data for the CodeGenerator tree."""
+
+  def __init__(self, filename, scopes, futures=()):
+    self.filename = filename
+    self.scopes = scopes
+    self.futures = futures
+
+
 # Emulating parser.st structures from parsermodule.c.
 # They have a totuple() method, which outputs tuples like this.
 def py2st(unused_gr, raw_node):
@@ -32,13 +43,52 @@ def py2st(unused_gr, raw_node):
    return (typ, value, lineno, column)


-class _ModuleContext(object):
-  """Module-level data for the CodeGenerator tree."""
-
-  def __init__(self, filename, scopes, futures=()):
-    self.filename = filename
-    self.scopes = scopes
-    self.futures = futures
+def MakeCodeObject(frame, graph):
+    """Order blocks, encode instructions, and create types.CodeType().
+
+    Called by RunCompiler below, and also recursively by ArgEncoder.
+    """
+    # Compute stack depth per basic block.
+    depths = {}
+    b = pyassem.BlockStackDepth()
+    for block in graph.blocks:
+        depths[block] = b.Sum(block.getInstructions())
+
+    # Compute maximum stack depth for any path through the CFG.
+    g = pyassem.GraphStackDepth(depths, graph.exit)
+    stacksize = g.Max(graph.entry, 0)
+
+    # Order blocks so jump offsets can be encoded.
+    blocks = pyassem.OrderBlocks(graph.entry, graph.exit)
+    insts = pyassem.FlattenGraph(blocks)
+
+    cellvars = pyassem.ReorderCellVars(frame)
+    consts = [frame.docstring]
+    names = []
+    # The closure list is used to track the order of cell variables and
+    # free variables in the resulting code object.  The offsets used by
+    # LOAD_CLOSURE/LOAD_DEREF refer to both kinds of variables.
+    closure = cellvars + frame.freevars
+
+    # Convert arguments from symbolic to concrete form.
+    enc = pyassem.ArgEncoder(frame.klass, consts, names, frame.varnames,
+                             closure)
+    # Mutates not only insts, but also consts, names, etc.
+    enc.Run(insts)
+
+    a = pyassem.Assembler()
+    bytecode, firstline, lnotab = a.Run(insts)
+
+    return types.CodeType(
+        frame.ArgCount(), frame.NumLocals(), stacksize, frame.flags,
+        bytecode,
+        tuple(consts),
+        tuple(names),
+        tuple(frame.varnames),
+        frame.filename, frame.name, firstline,
+        lnotab,
+        tuple(frame.freevars),
+        tuple(cellvars))


 def RunCompiler(f, filename, gr, start_symbol, mode):
@@ -54,39 +104,37 @@ def RunCompiler(f, filename, gr, start_symbol, mode):
  tr = transformer.Transformer()
  as_tree = tr.transform(parse_tree)

-  #log('AST: %s', as_tree)
-
  # NOTE: This currently does nothing!
  v = syntax.SyntaxErrorChecker()
  v.Dispatch(as_tree)

  s = symbols.SymbolVisitor()
  s.Dispatch(as_tree)

-  graph = pyassem.FlowGraph()
+  graph = pyassem.FlowGraph()  # Mutated by code generator
  if mode == "single":
-      # NOTE: the name of the flow graph is a comment, not exposed to users.
-      frame = pyassem.Frame("<interactive>", filename)
      ctx = _ModuleContext(filename, s.scopes)
-      gen = pycodegen.InteractiveCodeGenerator(frame, graph, ctx)
+      # NOTE: the name of the Frame is a comment, not exposed to users.
+      frame = pyassem.Frame("<interactive>", filename)  # mutated
+      gen = pycodegen.InteractiveCodeGenerator(ctx, frame, graph)
      gen.set_lineno(as_tree)

  elif mode == "exec":
-      frame = pyassem.Frame("<module>", filename)
-
      # TODO: Does this need to be made more efficient?
      p1 = future.FutureParser()
      p2 = future.BadFutureParser()
      p1.Dispatch(as_tree)
      p2.Dispatch(as_tree)

      ctx = _ModuleContext(filename, s.scopes, futures=p1.get_features())
-      gen = pycodegen.TopLevelCodeGenerator(frame, graph, ctx)
+      frame = pyassem.Frame("<module>", filename)  # mutated
+
+      gen = pycodegen.TopLevelCodeGenerator(ctx, frame, graph)

  elif mode == "eval":
-      frame = pyassem.Frame("<expression>", filename)
      ctx = _ModuleContext(filename, s.scopes)
-      gen = pycodegen.TopLevelCodeGenerator(frame, graph, ctx)
+      frame = pyassem.Frame("<expression>", filename)  # mutated
+      gen = pycodegen.TopLevelCodeGenerator(ctx, frame, graph)

  else:
      raise ValueError("compile() 3rd arg must be 'exec' or "
@@ -97,7 +145,5 @@ def RunCompiler(f, filename, gr, start_symbol, mode):
  gen.Finish()

  # NOTE: This method has a pretty long pipeline too.
-  co = pyassem.MakeCodeObject(gen.frame, gen.graph)
+  co = MakeCodeObject(frame, graph)
  return co
-
-