Fill out ovm_codegen.py.

This a stripped-down version of pycodegen.py. It contains only what we need to compile gold/fib_iterative.py. - Rename gold/fibonacci -> gold/fib_iterative. - Add gold/fib_recursive, and run it in callgraph mode too.
oilshell · Jul 1, 2018 · f0c349580145945600fc0df2e0a45e3533bfa2b4 · f0c3495
1 parent 6c0c9b9
commit f0c349580145945600fc0df2e0a45e3533bfa2b4
Unified Split

Showing with 244 additions and 31 deletions.

+168 −18 opy/compiler2/ovm_codegen.py

+4 −0 opy/count.sh

+2 −1 opy/gold/{fibonacci.py → fib_iterative.py}

+33 −0 opy/gold/fib_recursive.py

+10 −3 opy/opy_main.py

+27 −9 opy/test.sh
diff --git a/opy/compiler2/ovm_codegen.py b/opy/compiler2/ovm_codegen.py
@@ -8,9 +8,18 @@
 from . import pyassem
 from . import pycodegen
 from .visitor import ASTVisitor
+from .pycodegen import LocalNameFinder
+from .consts import (
+    SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, SC_FREE, SC_CELL)

 Frame = pyassem.Frame
-#CodeGenerator = pycodegen.TopLevelCodeGenerator
+
+# Block type?
+LOOP = 1
+
+
+def is_constant_false(node):
+    return isinstance(node, ast.Const) and not node.value


 class CodeGenerator(ASTVisitor):
@@ -21,6 +30,37 @@ def __init__(self, ctx, frame, graph):
        self.frame = frame
        self.graph = graph

+        self.locals = pycodegen.Stack()
+        self.setups = pycodegen.Stack()
+
+        self.emit = self.graph.emit
+        self.newBlock = self.graph.newBlock
+        self.startBlock = self.graph.startBlock
+        self.nextBlock = self.graph.nextBlock
+
+        self.last_lineno = None
+
+    def set_lineno(self, node, force=False):
+        """Emit SET_LINENO if necessary.
+
+        The instruction is considered necessary if the node has a
+        lineno attribute and it is different than the last lineno
+        emitted.
+
+        Returns true if SET_LINENO was emitted.
+
+        There are no rules for when an AST node should have a lineno
+        attribute.  The transformer and AST code need to be reviewed
+        and a consistent policy implemented and documented.  Until
+        then, this method works around missing line numbers.
+        """
+        lineno = getattr(node, 'lineno', None)
+        if lineno is not None and (lineno != self.last_lineno or force):
+            self.emit('SET_LINENO', lineno)
+            self.last_lineno = lineno
+            return True
+        return False
+
    def _Default(self, node):
        raise AssertionError('%s is unhandled' % node.__class__)

@@ -36,13 +76,25 @@ def visitFrom(self, node):
      pass

    def visitDiscard(self, node):
+        self.set_lineno(node)
        self.visit(node.expr)
-        #self.emit('POP_TOP')
+        self.emit('POP_TOP')

    def visitModule(self, node):
        print('Module')
-        print(node)
+        self.scope = self.ctx.scopes[node]
+        self.emit('SET_LINENO', 0)
+        if node.doc:
+            self.emit('LOAD_CONST', node.doc)
+            self.storeName('__doc__')
+
+        lnf = LocalNameFinder()
+        lnf.Dispatch(node.node)
+
+        self.locals.push(lnf.getLocals())
        self.visit(node.node)
+        self.emit('LOAD_CONST', None)
+        self.emit('RETURN_VALUE')

    def visitStmt(self, node):
        print('Stmt')
@@ -51,42 +103,128 @@ def visitStmt(self, node):

    def visitWhile(self, node):
        print('While')
+        self.set_lineno(node)
+
+        loop = self.newBlock()
+        else_ = self.newBlock()
+
+        after = self.newBlock()
+        self.emit('SETUP_LOOP', after)
+
+        self.nextBlock(loop)
+        self.setups.push((LOOP, loop))
+
+        self.set_lineno(node, force=True)
        self.visit(node.test)
+        self.emit('POP_JUMP_IF_FALSE', else_ or after)
+
+        self.nextBlock()
        self.visit(node.body)
+        self.emit('JUMP_ABSOLUTE', loop)
+
+        self.startBlock(else_) # or just the POPs if not else clause
+        self.emit('POP_BLOCK')
+        self.setups.pop()
        if node.else_:
-            raise AssertionError('else not allowed')
+            self.visit(node.else_)
+        self.nextBlock(after)

    def visitIf(self, node):
        print('If')
-        #print(dir(node))
+        end = self.newBlock()
        for i, (test, suite) in enumerate(node.tests):
+            if is_constant_false(test):
+                # XXX will need to check generator stuff here
+                continue
+            self.set_lineno(test)
            self.visit(test)
+            nextTest = self.newBlock()
+            self.emit('POP_JUMP_IF_FALSE', nextTest)
+            self.nextBlock()
            self.visit(suite)
+            self.emit('JUMP_FORWARD', end)
+            self.startBlock(nextTest)
        if node.else_:
            self.visit(node.else_)
+        self.nextBlock(end)

    def visitBreak(self, node):
        print('Break')
+        if not self.setups:
+            raise SyntaxError(
+                "'break' outside loop (%s, %d)" % 
+                (self.ctx.filename, node.lineno))
+        self.set_lineno(node)
+        self.emit('BREAK_LOOP')
+
+    def visitContinue(self, node):
+        print('Continue')
+        if not self.setups:
+            raise SyntaxError(
+                "'continue' outside loop (%s, %d)" %
+                (self.ctx.filename, node.lineno))
+
+        kind, block = self.setups.top()
+        if kind == LOOP:
+            self.set_lineno(node)
+            self.emit('JUMP_ABSOLUTE', block)
+            self.nextBlock()
+        else:
+            msg = "'continue' not handled here (%s, %d)"
+            raise SyntaxError(msg % (self.ctx.filename, node.lineno))
+
+    def _nameOp(self, prefix, name):
+        # Don't mangle
+        #name = self._mangle(name)
+        scope = self.scope.check_name(name)
+
+        if scope == SC_LOCAL:
+            #suffix = 'FAST' if self._optimized() else 'NAME'
+            # TODO: LOAD_FAST
+            suffix = 'NAME'
+            self.emit('%s_%s' % (prefix, suffix), name)
+
+        elif scope == SC_GLOBAL_EXPLICIT:
+            self.emit(prefix + '_GLOBAL', name)
+
+        elif scope == SC_GLOBAL_IMPLICIT:
+            #suffix = 'GLOBAL' if self._optimized() else 'NAME'
+            # TODO: LOAD_GLOBAL
+            suffix = 'NAME'
+            self.emit('%s_%s' % (prefix, suffix), name)
+
+        elif scope == SC_FREE or scope == SC_CELL:
+            self.emit(prefix + '_DEREF', name)
+
+        else:
+            raise RuntimeError, "unsupported scope for var %s: %d" % \
+                  (name, scope)
+
+    def loadName(self, name):
+        self._nameOp('LOAD', name)

    def visitName(self, node):
        print('Name')
-        #self.loadName(node.name)
-        pass
+        self.set_lineno(node)
+        self.loadName(node.name)
+
+    def storeName(self, name):
+        self._nameOp('STORE', name)

    def visitAssName(self, node):
        print('AssName')
        if node.flags == 'OP_ASSIGN':
-            #self.storeName(node.name)
-            pass
+            self.storeName(node.name)
        elif node.flags == 'OP_DELETE':
-            #self.delName(node.name)
-            pass
+            self.set_lineno(node)
+            self.delName(node.name)
        else:
            print("oops", node.flags)

    # When are there multiple assignments?
    def visitAssign(self, node):
        print('Assign')
+        self.set_lineno(node)
        self.visit(node.expr)
        dups = len(node.nodes) - 1
        for i, elt in enumerate(node.nodes):
@@ -98,17 +236,16 @@ def visitAssign(self, node):
    def binaryOp(self, node, op):
        self.visit(node.left)
        self.visit(node.right)
-        #self.emit(op)
+        self.emit(op)

    def visitAdd(self, node):
        print('Add')
        return self.binaryOp(node, 'BINARY_ADD')

    def visitCompare(self, node):
        print('Compare')
+        # I guess this is 1 < x < y < 3 
        self.visit(node.expr)
-        # I guess this is 1 < a < b < 2 ?
-        return
        cleanup = self.newBlock()
        for op, code in node.ops[:-1]:
            self.visit(code)
@@ -136,16 +273,29 @@ def visitCompare(self, node):
    def visitCallFunc(self, node):
        print('CallFunc')
        self.visit(node.node)
+        pos = 0
+        kw = 0
+        self.set_lineno(node)
+        self.visit(node.node)
        for arg in node.args:
            self.visit(arg)
-        # NOTE: Don't support these?  We do use *args, but they can be
-        # evaluated at compile-time?
+            if isinstance(arg, ast.Keyword):
+                kw += 1
+            else:
+                pos += 1
+        self.emit('CALL_FUNCTION', pos)
+
+        # For f(*args, **kwargs)
+        return
        if node.star_args is not None:
            self.visit(node.star_args)
        if node.dstar_args is not None:
            self.visit(node.dstar_args)
+        have_star = node.star_args is not None
+        have_dstar = node.dstar_args is not None
+        opcode = _CALLFUNC_OPCODE_INFO[have_star, have_dstar]
+        self.emit(opcode, kw << 8 | pos)

    def visitConst(self, node):
        print('Const')
-        #self.emit('LOAD_CONST', node.value)
-        pass
+        self.emit('LOAD_CONST', node.value)
diff --git a/opy/count.sh b/opy/count.sh
@@ -52,6 +52,10 @@ all() {
  echo
 }

+ovm() {
+  wc -l */ovm*.py
+}
+
 # Hm there are 119 total opcodes, but these files only use 38, 37, 36, and 23.
 # Interesting.

diff --git a/opy/gold/fibonacci.py → opy/gold/fib_iterative.py b/opy/gold/fibonacci.py → opy/gold/fib_iterative.py
@@ -1,5 +1,6 @@
 #!/usr/bin/python
 from __future__ import print_function
+"""Iterative version if Fibonacci."""

 i = 0
 n = 10
@@ -26,4 +27,4 @@
  if i == n:
    break

-print('Done')  # To make sure we implemented 'break' properly
+print('Done fib_iterative.py')  # To make sure we implemented 'break' properly
diff --git a/opy/gold/fib_recursive.py b/opy/gold/fib_recursive.py
@@ -0,0 +1,33 @@
+#!/usr/bin/python
+from __future__ import print_function
+"""Recursive version if Fibonacci."""
+
+
+def unused():
+  """A function that shouldn't be compiled."""
+  return 42
+
+
+def fib(n):
+  if n == 0:
+    return 1
+  elif n == 1:
+    return 1
+  else:
+    return fib(n-1) + fib(n-2)
+
+
+def main():
+  for i in xrange(9):
+    print(fib(i))
+  print('Done fib_recursive.py')
+
+
+if __name__ == '__main__':
+  import os
+  if os.getenv('CALLGRAPH') == '1':
+    import sys
+    from opy import callgraph
+    callgraph.Walk(main, sys.modules)
+  else:
+    main()
diff --git a/opy/opy_main.py b/opy/opy_main.py
@@ -246,7 +246,12 @@ def OpyCommandMain(argv):
    log("Compiled to %d bytes of bytecode", len(co.co_code))
    # Write the .pyc file
    with open(out_path, 'wb') as out_f:
-      out_f.write(co.co_code)
+      if 1:
+        out_f.write(co.co_code)
+      else:
+        h = misc.getPycHeader(py_path)
+        out_f.write(h)
+        marshal.dump(co, out_f)
    log('Wrote only the bytecode to %r', out_path)

  elif action == 'eval':  # Like compile, but parses to a code object and prints it
@@ -338,9 +343,11 @@ def OpyCommandMain(argv):
    opy_argv = argv[1:]

    if py_path.endswith('.py'):
-      # TODO: use ovm mode
+      #mode = 'exec'
+      mode = 'ovm'  # OVM bytecode is different!
      with open(py_path) as f:
-        co = skeleton.Compile(f, py_path, gr, 'file_input', 'exec')
+        co = skeleton.Compile(f, py_path, gr, 'file_input', mode)
+      log('Compiled to %d bytes of OVM code', len(co.co_code))
      num_ticks = ovm.run_code_object(co, opy_argv)

    elif py_path.endswith('.pyc') or py_path.endswith('.opyc'):