View
260 opy/compiler2/transformer.py
@@ -26,9 +26,140 @@
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
from compiler2.ast import *
import parser
import symbol
import token
#import parser # stdlib module we want to get rid of
from pgen2 import token
#import token
from pytree import type_repr
symbol = None
def Init(sym):
"""Replacement for the stdlib symbol module.
Args:
sym: module
The stdlib module is generated from pgen.c's output data. pgen2 derives it
from the grammar directly.
"""
global symbol
symbol = sym
_InitGlobals()
# Various constants
_doc_nodes = []
_legal_node_types = []
_assign_types = []
# NOTE: This is somewhat duplicated in pytree.py as type_repr.
_names = {}
def _InitGlobals():
_doc_nodes.extend([
symbol.expr_stmt,
symbol.testlist,
symbol.testlist_safe,
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
])
_legal_node_types.extend([
symbol.funcdef,
symbol.classdef,
symbol.stmt,
symbol.small_stmt,
symbol.flow_stmt,
symbol.simple_stmt,
symbol.compound_stmt,
symbol.expr_stmt,
symbol.print_stmt,
symbol.del_stmt,
symbol.pass_stmt,
symbol.break_stmt,
symbol.continue_stmt,
symbol.return_stmt,
symbol.raise_stmt,
symbol.import_stmt,
symbol.global_stmt,
symbol.exec_stmt,
symbol.assert_stmt,
symbol.if_stmt,
symbol.while_stmt,
symbol.for_stmt,
symbol.try_stmt,
symbol.with_stmt,
symbol.suite,
symbol.testlist,
symbol.testlist_safe,
symbol.test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.exprlist,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
symbol.atom,
])
if hasattr(symbol, 'yield_stmt'):
_legal_node_types.append(symbol.yield_stmt)
if hasattr(symbol, 'yield_expr'):
_legal_node_types.append(symbol.yield_expr)
_assign_types.extend([
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
])
# Do this first beause NT_OFFSET (non-terminal offset) conflicts with
# file_input.
for k, v in token.tok_name.items():
_names[k] = v
for k, v in symbol.number2symbol.items():
_names[k] = v
# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
# | 'in' | 'not' 'in' | 'is' | 'is' 'not'
_cmp_types = {
token.LESS : '<',
token.GREATER : '>',
token.EQEQUAL : '==',
token.EQUAL : '==',
token.LESSEQUAL : '<=',
token.GREATEREQUAL : '>=',
token.NOTEQUAL : '!=',
}
class WalkerError(StandardError):
pass
@@ -46,11 +177,12 @@ def parseFile(path):
f.close()
return parse(src)
def parse(buf, mode="exec"):
def parse(buf, mode="exec", transformer=None):
tr = transformer or Transformer()
if mode == "exec" or mode == "single":
return Transformer().parsesuite(buf)
return tr.parsesuite(buf)
elif mode == "eval":
return Transformer().parseexpr(buf)
return tr.parseexpr(buf)
else:
raise ValueError("compile() arg 3 must be"
" 'exec' or 'eval' or 'single'")
@@ -103,7 +235,7 @@ class Transformer:
def __init__(self):
self._dispatch = {}
for value, name in symbol.sym_name.items():
for value, name in symbol.number2symbol.items():
if hasattr(self, name):
self._dispatch[value] = getattr(self, name)
self._dispatch[token.NEWLINE] = self.com_NEWLINE
@@ -119,8 +251,9 @@ def __init__(self):
def transform(self, tree):
"""Transform an AST into a modified parse tree."""
if not (isinstance(tree, tuple) or isinstance(tree, list)):
tree = parser.st2tuple(tree, line_info=1)
# We only use Pgen2Transformer
# if not (isinstance(tree, tuple) or isinstance(tree, list)):
# tree = parser.st2tuple(tree, line_info=1)
return self.compile_node(tree)
def parsesuite(self, text):
@@ -164,7 +297,7 @@ def compile_node(self, node):
if n == symbol.classdef:
return self.classdef(node[1:])
raise WalkerError, ('unexpected node type', n)
raise WalkerError('unexpected node type %r' % type_repr(n))
def single_input(self, node):
### do we want to do anything about being "interactive" ?
@@ -1422,106 +1555,17 @@ def get_docstring(self, node, n=None):
return None
_doc_nodes = [
symbol.expr_stmt,
symbol.testlist,
symbol.testlist_safe,
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
]
# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
# | 'in' | 'not' 'in' | 'is' | 'is' 'not'
_cmp_types = {
token.LESS : '<',
token.GREATER : '>',
token.EQEQUAL : '==',
token.EQUAL : '==',
token.LESSEQUAL : '<=',
token.GREATEREQUAL : '>=',
token.NOTEQUAL : '!=',
}
_legal_node_types = [
symbol.funcdef,
symbol.classdef,
symbol.stmt,
symbol.small_stmt,
symbol.flow_stmt,
symbol.simple_stmt,
symbol.compound_stmt,
symbol.expr_stmt,
symbol.print_stmt,
symbol.del_stmt,
symbol.pass_stmt,
symbol.break_stmt,
symbol.continue_stmt,
symbol.return_stmt,
symbol.raise_stmt,
symbol.import_stmt,
symbol.global_stmt,
symbol.exec_stmt,
symbol.assert_stmt,
symbol.if_stmt,
symbol.while_stmt,
symbol.for_stmt,
symbol.try_stmt,
symbol.with_stmt,
symbol.suite,
symbol.testlist,
symbol.testlist_safe,
symbol.test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.exprlist,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
symbol.atom,
]
if hasattr(symbol, 'yield_stmt'):
_legal_node_types.append(symbol.yield_stmt)
if hasattr(symbol, 'yield_expr'):
_legal_node_types.append(symbol.yield_expr)
class Pgen2Transformer(Transformer):
def __init__(self, py_parser, printer):
Transformer.__init__(self)
self.py_parser = py_parser
self.printer = printer
_assign_types = [
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
]
def parsesuite(self, text):
tree = self.py_parser.suite(text)
#self.printer.Print(tree)
return self.transform(tree)
_names = {}
for k, v in symbol.sym_name.items():
_names[k] = v
for k, v in token.tok_name.items():
_names[k] = v
def debug_tree(tree):
l = []
View
94 opy/opy_main.py
@@ -4,6 +4,7 @@
"""
from __future__ import print_function
import cStringIO
import codecs
import io
import os
@@ -15,13 +16,14 @@
import pytree
# TODO: Delete these
from compiler import transformer
from compiler import pycodegen
from compiler import opcode27
#from compiler2 import transformer
#from compiler2 import pycodegen
#from compiler2 import opcode
if 0:
from compiler import transformer
from compiler import pycodegen
from compiler import opcode27
else:
from compiler2 import transformer
from compiler2 import pycodegen
from compiler2 import opcode
from util import log
import util
@@ -72,17 +74,51 @@ def __init__(self, driver, start_symbol):
self.start_symbol = start_symbol
def suite(self, text):
#if util.PY2:
if _READ_SOURCE_AS_UNICODE:
f = io.StringIO(text)
else:
import cStringIO
f = cStringIO.StringIO()
# Python 3
#f = io.StringIO(text)
f = cStringIO.StringIO(text)
tokens = tokenize.generate_tokens(f.readline)
tree = self.driver.parse_tokens(tokens, start_symbol=self.start_symbol)
return tree
def CountTupleTree(tu):
"""Count the nodes in a tuple parse tree."""
if isinstance(tu, tuple):
s = 0
for entry in tu:
s += CountTupleTree(entry)
return s
elif isinstance(tu, int):
return 1
elif isinstance(tu, str):
return 1
else:
raise AssertionError(tu)
class TupleTreePrinter:
def __init__(self, names):
self._names = names
def Print(self, tu, f=sys.stdout, indent=0):
ind = ' ' * indent
f.write(ind)
if isinstance(tu, tuple):
f.write(self._names[tu[0]])
f.write('\n')
for entry in tu[1:]:
self.Print(entry, f, indent=indent+1)
elif isinstance(tu, int):
f.write(str(tu))
f.write('\n')
elif isinstance(tu, str):
f.write(str(tu))
f.write('\n')
else:
raise AssertionError(tu)
def main(argv):
grammar_path = argv[1]
# NOTE: This is cached as a pickle
@@ -138,7 +174,7 @@ def py2st(grammar, raw_node):
n = transformer.CountTupleTree(tree)
log('COUNT %d', n)
printer = transformer.TupleTreePrinter(HostStdlibNames())
printer = TupleTreePrinter(HostStdlibNames())
printer.Print(tree)
elif action == 'parse':
@@ -148,10 +184,10 @@ def py2st(grammar, raw_node):
tree = d.parse_tokens(tokens, start_symbol=FILE_INPUT)
if isinstance(tree, tuple):
n = transformer.CountTupleTree(tree)
n = CountTupleTree(tree)
log('COUNT %d', n)
printer = transformer.TupleTreePrinter(transformer._names)
printer = TupleTreePrinter(transformer._names)
printer.Print(tree)
else:
tree.PrettyPrint(sys.stdout)
@@ -163,7 +199,7 @@ def py2st(grammar, raw_node):
if do_glue:
py_parser = Pgen2PythonParser(d, FILE_INPUT)
printer = transformer.TupleTreePrinter(transformer._names)
printer = TupleTreePrinter(transformer._names)
tr = transformer.Pgen2Transformer(py_parser, printer)
else:
tr = transformer.Transformer()
@@ -176,7 +212,6 @@ def py2st(grammar, raw_node):
contents = f.read()
co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
file_size = os.path.getsize(py_path)
log("Code length: %d", len(co.co_code))
# Write the .pyc file
@@ -188,7 +223,28 @@ def py2st(grammar, raw_node):
elif action == 'compile':
# 'opy compile' is pgen2 + compiler2
# TODO: import compiler2
raise NotImplementedError
#raise NotImplementedError
py_path = argv[3]
out_path = argv[4]
if do_glue:
py_parser = Pgen2PythonParser(d, FILE_INPUT)
printer = TupleTreePrinter(transformer._names)
tr = transformer.Pgen2Transformer(py_parser, printer)
else:
tr = transformer.Transformer()
f = open(py_path)
contents = f.read()
co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
log("Code length: %d", len(co.co_code))
# Write the .pyc file
with open(out_path, 'wb') as out_f:
h = pycodegen.getPycHeader(py_path)
out_f.write(h)
marshal.dump(co, out_f)
elif action == 'compile2':
in_path = argv[3]
View
9 opy/run.sh
@@ -18,7 +18,8 @@ die() {
}
_parse-one() {
PYTHONPATH=. ./opy_main.py 2to3.grammar parse "$@"
#PYTHONPATH=. ./opy_main.py 2to3.grammar parse "$@"
PYTHONPATH=. ./opy_main.py py27.grammar parse "$@"
}
parse-test() {
@@ -54,7 +55,13 @@ _compile-and-run() {
mkdir -p _tmp
local out=_tmp/${basename}.pyc
#_parse-one $path
# new opy compile
_compile-one $path $out
# unmodified pgen2
#_compile2-one $path $out
ls -l $out
xxd $out
View
3 opy/smoke.sh
@@ -75,7 +75,8 @@ compile-osh-tree() {
-name '*.py' -a -printf '%P\n') )
_compile-tree $src _tmp/osh-ccompile/ ccompile "${files[@]}"
_compile-tree $src _tmp/osh-compile2/ compiler2 "${files[@]}"
#_compile-tree $src _tmp/osh-compile2/ compiler2 "${files[@]}"
_compile-tree $src _tmp/osh-opy/ opy "${files[@]}"
# Not deterministic!
#_compile-tree $src _tmp/osh-compile2.gold/ compiler2 "${files[@]}"
View
4 tests/loop.test.sh
@@ -111,12 +111,12 @@ echo $i
### while in pipe with subshell
i=0
find bin/ -type f | ( while read path; do
find bin/ -type l | ( while read path; do
i=$((i+1))
#echo $i
done
echo $i )
# stdout: 2
# stdout: 5
### until loop
# This is just the opposite of while? while ! cond?