View

Large diffs are not rendered by default.

Oops, something went wrong.
View
@@ -5,6 +5,7 @@
from __future__ import print_function, division
import linecache
import operator
import os
import repr as repr_lib # Don't conflict with builtin repr()
import sys
import traceback
@@ -86,6 +87,9 @@ def run_code(vm, code, f_globals=None):
frame = vm.make_frame(code, f_globals=f_globals)
val = vm.run_frame(frame)
vm.check_invariants()
if os.getenv('BYTERUN_SUMMARY'):
debug1('*** Byterun executed for %d ticks', vm.num_ticks)
# If we return the number of ticks here, the unit tests break.
return val
@@ -116,6 +120,7 @@ def __init__(self, subset=False, verbose=VERBOSE):
self.last_exception = None
self.except_frames = [] # Frames saved for GuestException
self.cur_line = None # current line number
self.num_ticks = 0
def top(self):
return self.frame.top()
@@ -169,7 +174,7 @@ def resume_frame(self, frame):
frame.f_back = None
return val
def logTick(self, byteName, arguments, opoffset, linestarts):
def log_tick(self, byteName, arguments, opoffset, linestarts):
""" Log arguments, block stack, and data stack for each opcode."""
indent = " " * (len(self.frames)-1)
stack_rep = repper(self.frame.stack)
@@ -263,14 +268,13 @@ def run_frame(self, frame):
#print('STARTS %s ' % linestarts)
self._push_frame(frame)
num_ticks = 0
while True:
num_ticks += 1
self.num_ticks += 1
opoffset = self.frame.f_lasti # For logging only
byteName, arguments = self.frame.decode_next()
if self.verbose:
self.logTick(byteName, arguments, opoffset, linestarts)
self.log_tick(byteName, arguments, opoffset, linestarts)
# When unwinding the block stack, we need to keep track of why we
# are doing it.
@@ -849,13 +853,13 @@ def byte_CALL_FUNCTION_VAR_KW(self, arg):
return self.call_function(arg, args, kwargs)
def call_function(self, arg, args, kwargs):
lenKw, lenPos = divmod(arg, 256)
len_kw, len_pos = divmod(arg, 256)
namedargs = {}
for i in range(lenKw):
for i in xrange(len_kw):
key, val = self.popn(2)
namedargs[key] = val
namedargs.update(kwargs)
posargs = self.popn(lenPos)
posargs = self.popn(len_pos)
posargs.extend(args)
#debug('*** call_function stack = %s', self.frame.stack)
View
@@ -0,0 +1,151 @@
#!/usr/bin/python
from __future__ import print_function
"""
ovm_codegen.py
"""
from . import ast
from . import pyassem
from . import pycodegen
from .visitor import ASTVisitor
Frame = pyassem.Frame
#CodeGenerator = pycodegen.TopLevelCodeGenerator
class CodeGenerator(ASTVisitor):
def __init__(self, ctx, frame, graph):
ASTVisitor.__init__(self)
self.ctx = ctx # passed down to child CodeGenerator instances
self.frame = frame
self.graph = graph
def _Default(self, node):
raise AssertionError('%s is unhandled' % node.__class__)
def Start(self):
# NOTE: Not used at the top level?
print('Start')
def Finish(self):
print('Finish')
# Ignore imports
def visitFrom(self, node):
pass
def visitDiscard(self, node):
self.visit(node.expr)
#self.emit('POP_TOP')
def visitModule(self, node):
print('Module')
print(node)
self.visit(node.node)
def visitStmt(self, node):
print('Stmt')
for child in node.nodes:
self.visit(child)
def visitWhile(self, node):
print('While')
self.visit(node.test)
self.visit(node.body)
if node.else_:
raise AssertionError('else not allowed')
def visitIf(self, node):
print('If')
#print(dir(node))
for i, (test, suite) in enumerate(node.tests):
self.visit(test)
self.visit(suite)
if node.else_:
self.visit(node.else_)
def visitBreak(self, node):
print('Break')
def visitName(self, node):
print('Name')
#self.loadName(node.name)
pass
def visitAssName(self, node):
print('AssName')
if node.flags == 'OP_ASSIGN':
#self.storeName(node.name)
pass
elif node.flags == 'OP_DELETE':
#self.delName(node.name)
pass
else:
print("oops", node.flags)
# When are there multiple assignments?
def visitAssign(self, node):
print('Assign')
self.visit(node.expr)
dups = len(node.nodes) - 1
for i, elt in enumerate(node.nodes):
if i < dups:
self.emit('DUP_TOP')
if isinstance(elt, ast.Node):
self.visit(elt)
def binaryOp(self, node, op):
self.visit(node.left)
self.visit(node.right)
#self.emit(op)
def visitAdd(self, node):
print('Add')
return self.binaryOp(node, 'BINARY_ADD')
def visitCompare(self, node):
print('Compare')
self.visit(node.expr)
# I guess this is 1 < a < b < 2 ?
return
cleanup = self.newBlock()
for op, code in node.ops[:-1]:
self.visit(code)
self.emit('DUP_TOP')
self.emit('ROT_THREE')
self.emit('COMPARE_OP', op)
self.emit('JUMP_IF_FALSE_OR_POP', cleanup)
self.nextBlock()
# now do the last comparison
if node.ops:
op, code = node.ops[-1]
self.visit(code)
self.emit('COMPARE_OP', op)
if len(node.ops) > 1:
end = self.newBlock()
self.emit('JUMP_FORWARD', end)
self.startBlock(cleanup)
self.emit('ROT_TWO')
self.emit('POP_TOP')
self.nextBlock(end)
# For Fibonacci, this is always print().
# TODO: Look up type or at least arity statically?
def visitCallFunc(self, node):
print('CallFunc')
self.visit(node.node)
for arg in node.args:
self.visit(arg)
# NOTE: Don't support these? We do use *args, but they can be
# evaluated at compile-time?
if node.star_args is not None:
self.visit(node.star_args)
if node.dstar_args is not None:
self.visit(node.dstar_args)
def visitConst(self, node):
print('Const')
#self.emit('LOAD_CONST', node.value)
pass
View
@@ -6,7 +6,13 @@
a = 0
b = 1
while True:
while 1: # Slightly easier to compile than 'while True:'
# Artifical change to test 'continue'
if i == 0:
i = i + 1
continue
print(b)
# NOTE: This would generate BUILD_TUPLE and UNPACK_SEQUENCE bytecodes.
@@ -16,6 +22,8 @@
a = b
b = tmp + b
i += 1
i = i + 1 # Don't use augmented assignment
if i == n:
break
print('Done') # To make sure we implemented 'break' properly
View
@@ -24,6 +24,7 @@
# Disabled for now because byterun imports 'six', and that breaks the build.
from .byterun import execfile
from .byterun import ovm
from core import args
from core import util
@@ -121,7 +122,7 @@ def Options():
# TODO: more actions:
# - lex, parse, ast, cfg, compile/eval/repl
# - lex, parse, ast, compile/eval/repl
# Made by the Makefile.
PICKLE_REL_PATH = '_build/opy/py27.grammar.pickle'
@@ -137,7 +138,9 @@ def OpyCommandMain(argv):
except IndexError:
raise args.UsageError('opy: Missing required subcommand.')
if action in ('parse', 'compile', 'compile-fib', 'eval', 'repl', 'run'):
if action in (
'parse', 'compile', 'cfg', 'compile-ovm', 'eval', 'repl', 'run',
'run-ovm'):
loader = util.GetResourceLoader()
f = loader.open(PICKLE_REL_PATH)
gr = grammar.Grammar()
@@ -209,6 +212,14 @@ def OpyCommandMain(argv):
tree.PrettyPrint(sys.stdout)
log('\tChildren: %d' % len(tree.children), file=sys.stderr)
elif action == 'cfg': # output fg
py_path = argv[1]
with open(py_path) as f:
graph = skeleton.Compile(f, py_path, gr, 'file_input', 'exec',
return_cfg=True)
print(graph)
elif action == 'compile': # 'opyc compile' is pgen2 + compiler2
py_path = argv[1]
out_path = argv[2]
@@ -224,12 +235,13 @@ def OpyCommandMain(argv):
out_f.write(h)
marshal.dump(co, out_f)
elif action == 'compile-fib':
elif action == 'compile-ovm':
py_path = argv[1]
out_path = argv[2]
# Compile to OVM bytecode
with open(py_path) as f:
co = skeleton.Compile(f, py_path, gr, 'file_input', 'exec')
co = skeleton.Compile(f, py_path, gr, 'file_input', 'ovm')
log("Compiled to %d bytes of bytecode", len(co.co_code))
# Write the .pyc file
@@ -309,13 +321,33 @@ def OpyCommandMain(argv):
if py_path.endswith('.py'):
with open(py_path) as f:
co = skeleton.Compile(f, py_path, gr, 'file_input', 'exec')
execfile.run_code_object(co, opy_argv)
num_ticks = execfile.run_code_object(co, opy_argv)
elif py_path.endswith('.pyc') or py_path.endswith('.opyc'):
with open(py_path) as f:
f.seek(8) # past header. TODO: validate it!
co = marshal.load(f)
num_ticks = execfile.run_code_object(co, opy_argv)
else:
raise args.UsageError('Invalid path %r' % py_path)
elif action == 'run-ovm':
# Compile and run, without writing pyc file
py_path = argv[1]
opy_argv = argv[1:]
if py_path.endswith('.py'):
# TODO: use ovm mode
with open(py_path) as f:
co = skeleton.Compile(f, py_path, gr, 'file_input', 'exec')
num_ticks = ovm.run_code_object(co, opy_argv)
elif py_path.endswith('.pyc') or py_path.endswith('.opyc'):
with open(py_path) as f:
f.seek(8) # past header. TODO: validate it!
co = marshal.load(f)
execfile.run_code_object(co, opy_argv)
num_ticks = ovm.run_code_object(co, opy_argv)
else:
raise args.UsageError('Invalid path %r' % py_path)
View
@@ -10,6 +10,7 @@
from .compiler2 import future
from .compiler2 import pyassem
from .compiler2 import pycodegen
from .compiler2 import ovm_codegen
from .compiler2 import syntax
from .compiler2 import symbols
from .compiler2 import transformer
@@ -41,7 +42,7 @@ def py2st(unused_gr, raw_node):
return (typ, value, lineno, column)
def Compile(f, filename, gr, start_symbol, mode):
def Compile(f, filename, gr, start_symbol, mode, return_cfg=False):
"""Run the full compiler pipeline.
Args:
@@ -50,6 +51,7 @@ def Compile(f, filename, gr, start_symbol, mode):
gr: Grammar
start_symbol: name of the grammar start symbol
mode: 'exec', 'eval', or 'single', like Python's builtin compile()
return_cfg: A little hack to stop at the CFG stage
"""
tokens = tokenize.generate_tokens(f.readline)
@@ -92,14 +94,22 @@ def Compile(f, filename, gr, start_symbol, mode):
frame = pyassem.Frame("<expression>", filename) # mutated
gen = pycodegen.TopLevelCodeGenerator(ctx, frame, graph)
elif mode == "ovm":
ctx = _ModuleContext(filename, s.scopes)
frame = ovm_codegen.Frame("<expression>", filename) # mutated
gen = ovm_codegen.CodeGenerator(ctx, frame, graph)
else:
raise AssertionError('Invalid mode %r' % mode)
# NOTE: There is no Start() or FindLocals() at the top level.
gen.Dispatch(as_tree) # mutates graph
gen.Finish()
if return_cfg:
return graph
co = pyassem.MakeCodeObject(frame, graph)
# TODO: Could call marshal.dump here?
# NOTE: Could call marshal.dump here?
return co
View
@@ -239,6 +239,11 @@ fib-dis() {
../bin/opyc dis $pyc
}
# TODO: Show graph output
fib-cfg() {
../bin/opyc cfg gold/fibonacci.py
}
# TODO: Move this to the OVM dir
run-ovm() {
local bin=_tmp/ovm_main
@@ -252,10 +257,21 @@ run-ovm() {
$bin "$@"
}
compile-fib() {
fib-ovm-prototype() {
local bytecode=_tmp/fibonacci.bytecode
../bin/opyc compile-fib gold/fibonacci.py $bytecode
VM_SUMMARY=1 ../bin/opyc run-ovm gold/fibonacci.py
}
fib-ovm-native() {
local bytecode=_tmp/fibonacci.bytecode
../bin/opyc compile-ovm gold/fibonacci.py $bytecode
run-ovm $bytecode
}
fib-byterun() {
local bytecode=_tmp/fibonacci.pyc
../bin/opyc compile gold/fibonacci.py $bytecode
BYTERUN_SUMMARY=1 ../bin/opyc run $bytecode
}
"$@"