View
322 opy/compiler/transformer.py
@@ -25,10 +25,147 @@
# http://www.opensource.org/licenses/bsd-license.html
# and replace OWNER, ORGANIZATION, and YEAR as appropriate.
import sys
from compiler.ast import *
import parser
import symbol
import token
from pgen2 import token # has BACKQUOTE
from pytree import type_repr
symbol = None
def Init(sym):
"""Replacement for the stdlib symbol module.
Args:
sym: module
The stdlib module is generated from pgen.c's output data. pgen2 derives it
from the grammar directly.
"""
global symbol
symbol = sym
_InitGlobals()
# Various constants
_doc_nodes = []
_legal_node_types = []
_assign_types = []
# NOTE: This is somewhat duplicated in pytree.py as type_repr.
_names = {}
def _InitGlobals():
_doc_nodes.extend([
symbol.expr_stmt,
symbol.testlist,
# No longer in Python 3
#symbol.testlist_safe,
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
])
_legal_node_types.extend([
symbol.funcdef,
symbol.classdef,
symbol.stmt,
symbol.small_stmt,
symbol.flow_stmt,
symbol.simple_stmt,
symbol.compound_stmt,
symbol.expr_stmt,
# No longer in Python 3
#symbol.print_stmt,
symbol.del_stmt,
symbol.pass_stmt,
symbol.break_stmt,
symbol.continue_stmt,
symbol.return_stmt,
symbol.raise_stmt,
symbol.import_stmt,
symbol.global_stmt,
# No longer in Python 3
#symbol.exec_stmt,
symbol.assert_stmt,
symbol.if_stmt,
symbol.while_stmt,
symbol.for_stmt,
symbol.try_stmt,
symbol.with_stmt,
symbol.suite,
symbol.testlist,
# No longer in Python 3
#symbol.testlist_safe,
symbol.test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.exprlist,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
symbol.atom,
])
if hasattr(symbol, 'yield_stmt'):
_legal_node_types.append(symbol.yield_stmt)
if hasattr(symbol, 'yield_expr'):
_legal_node_types.append(symbol.yield_expr)
_assign_types.extend([
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
])
# Do this first beause NT_OFFSET (non-terminal offset) conflicts with
# file_input.
for k, v in token.tok_name.items():
_names[k] = v
for k, v in symbol.number2symbol.items():
_names[k] = v
# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
# | 'in' | 'not' 'in' | 'is' | 'is' 'not'
_cmp_types = {
token.LESS : '<',
token.GREATER : '>',
token.EQEQUAL : '==',
token.EQUAL : '==',
token.LESSEQUAL : '<=',
token.GREATEREQUAL : '>=',
token.NOTEQUAL : '!=',
}
class WalkerError(Exception):
pass
@@ -46,11 +183,15 @@ def parseFile(path):
f.close()
return parse(src)
def parse(buf, mode="exec"):
def parse(buf, mode="exec", transformer=None):
assert symbol is not None
#print('!!', transformer)
tr = transformer or Transformer()
if mode == "exec" or mode == "single":
return Transformer().parsesuite(buf)
return tr.parsesuite(buf)
elif mode == "eval":
return Transformer().parseexpr(buf)
return tr.parseexpr(buf)
else:
raise ValueError("compile() arg 3 must be"
" 'exec' or 'eval' or 'single'")
@@ -91,6 +232,43 @@ def Node(*args):
raise WalkerError("Can't find appropriate Node type: %s" % str(args))
#return apply(ast.Node, args)
def CountTupleTree(tu):
if isinstance(tu, tuple):
s = 0
for entry in tu:
s += CountTupleTree(entry)
return s
elif isinstance(tu, int):
return 1
elif isinstance(tu, str):
return 1
else:
raise AssertionError(tu)
class TupleTreePrinter:
def __init__(self, names):
self._names = names
def Print(self, tu, f=sys.stdout, indent=0):
ind = ' ' * indent
f.write(ind)
if isinstance(tu, tuple):
f.write(self._names[tu[0]])
f.write('\n')
for entry in tu[1:]:
self.Print(entry, f, indent=indent+1)
elif isinstance(tu, int):
f.write(str(tu))
f.write('\n')
elif isinstance(tu, str):
f.write(str(tu))
f.write('\n')
else:
raise AssertionError(tu)
class Transformer:
"""Utility object for transforming Python parse trees.
@@ -102,8 +280,9 @@ class Transformer:
"""
def __init__(self):
assert symbol is not None
self._dispatch = {}
for value, name in list(symbol.sym_name.items()):
for value, name in list(symbol.number2symbol.items()):
if hasattr(self, name):
self._dispatch[value] = getattr(self, name)
self._dispatch[token.NEWLINE] = self.com_NEWLINE
@@ -125,7 +304,21 @@ def transform(self, tree):
def parsesuite(self, text):
"""Return a modified parse tree for the given suite text."""
return self.transform(parser.suite(text))
# TODO: This returns stdlib structures: parser.st.
# https://docs.python.org/2/library/parser.html
# We overrode it to to use pytree.py structures created by pgen2.
st = parser.suite(text)
# Debugging
#print(st)
#print(dir(st))
#tu = st.totuple()
#from pprint import pprint
#print('COUNT', CountTupleTree(tu))
#pprint(tu)
return self.transform(st)
def parseexpr(self, text):
"""Return a modified parse tree for the given expression text."""
@@ -164,7 +357,7 @@ def compile_node(self, node):
if n == symbol.classdef:
return self.classdef(node[1:])
raise WalkerError('unexpected node type', n)
raise WalkerError('unexpected node type: %s' % type_repr(n))
def single_input(self, node):
### do we want to do anything about being "interactive" ?
@@ -779,7 +972,8 @@ def atom_string(self, nodelist):
return Const(k, lineno=nodelist[0][2])
def atom_name(self, nodelist):
return Name(nodelist[0][1], lineno=nodelist[0][2])
return Name(nodelist[0][1], lineno=nodelist[0][2])
# --------------------------------------------------------------
#
@@ -795,7 +989,8 @@ def atom_name(self, nodelist):
# dispatched node directly.
def lookup_node(self, node):
return self._dispatch[node[0]]
node_type = node[0]
return self._dispatch[node_type]
def com_node(self, node):
# Note: compile.c has handling in com_node for del_stmt, pass_stmt,
@@ -1420,106 +1615,17 @@ def get_docstring(self, node, n=None):
return None
_doc_nodes = [
symbol.expr_stmt,
symbol.testlist,
symbol.testlist_safe,
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
]
# comp_op: '<' | '>' | '=' | '>=' | '<=' | '<>' | '!=' | '=='
# | 'in' | 'not' 'in' | 'is' | 'is' 'not'
_cmp_types = {
token.LESS : '<',
token.GREATER : '>',
token.EQEQUAL : '==',
token.EQUAL : '==',
token.LESSEQUAL : '<=',
token.GREATEREQUAL : '>=',
token.NOTEQUAL : '!=',
}
class Pgen2Transformer(Transformer):
def __init__(self, py_parser, printer):
Transformer.__init__(self)
self.py_parser = py_parser
self.printer = printer
_legal_node_types = [
symbol.funcdef,
symbol.classdef,
symbol.stmt,
symbol.small_stmt,
symbol.flow_stmt,
symbol.simple_stmt,
symbol.compound_stmt,
symbol.expr_stmt,
symbol.print_stmt,
symbol.del_stmt,
symbol.pass_stmt,
symbol.break_stmt,
symbol.continue_stmt,
symbol.return_stmt,
symbol.raise_stmt,
symbol.import_stmt,
symbol.global_stmt,
symbol.exec_stmt,
symbol.assert_stmt,
symbol.if_stmt,
symbol.while_stmt,
symbol.for_stmt,
symbol.try_stmt,
symbol.with_stmt,
symbol.suite,
symbol.testlist,
symbol.testlist_safe,
symbol.test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.exprlist,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
symbol.power,
symbol.atom,
]
if hasattr(symbol, 'yield_stmt'):
_legal_node_types.append(symbol.yield_stmt)
if hasattr(symbol, 'yield_expr'):
_legal_node_types.append(symbol.yield_expr)
_assign_types = [
symbol.test,
symbol.or_test,
symbol.and_test,
symbol.not_test,
symbol.comparison,
symbol.expr,
symbol.xor_expr,
symbol.and_expr,
symbol.shift_expr,
symbol.arith_expr,
symbol.term,
symbol.factor,
]
def parsesuite(self, text):
tree = self.py_parser.suite(text)
#self.printer.Print(tree)
return self.transform(tree)
_names = {}
for k, v in list(symbol.sym_name.items()):
_names[k] = v
for k, v in list(token.tok_name.items()):
_names[k] = v
def debug_tree(tree):
l = []
View
169 opy/parse.py
@@ -4,26 +4,78 @@
"""
import io
import os
import sys
from opy.pgen2 import driver
from opy.pgen2 import token, tokenize
from opy import pytree
from opy import pygram
from pgen2 import driver
from pgen2 import token, tokenize
import pytree
from compiler import transformer
from compiler import pycodegen
# From lib2to3/pygram.py. This presumably takes the place of the 'symbol'
# module. Could we hook it up elsewhere?
#
# compiler/transformer module needs this.
# tools/compile.py runs compileFile, which runs parseFile.
class Symbols(object):
def __init__(self, grammar):
"""Initializer.
Creates an attribute for each grammar symbol (nonterminal),
whose value is the symbol's type (an int >= 256).
"""
for name, symbol in grammar.symbol2number.items():
setattr(self, name, symbol)
#print('%s -> %d' % (name, symbol))
# For transformer to use
self.number2symbol = grammar.number2symbol
#assert 0
def HostStdlibNames():
import symbol
import token
names = {}
for k, v in symbol.sym_name.items():
names[k] = v
for k, v in token.tok_name.items():
names[k] = v
return names
# Emulate the interface that Transformer expects from parsermodule.c.
class Pgen2PythonParser:
def __init__(self, driver, start_symbol):
self.driver = driver
self.start_symbol = start_symbol
def suite(self, text):
f = io.StringIO(text)
tokens = tokenize.generate_tokens(f.readline)
tree = self.driver.parse_tokens(tokens, start_symbol=self.start_symbol)
return tree
def main(argv):
grammar_path = argv[1]
# NOTE: This is cached as a pickle
grammar = driver.load_grammar(grammar_path)
FILE_INPUT = grammar.symbol2number['file_input']
# NOTE: Parsing tables are cached with pickle.
# For Python 3.
# lib2to3 had a flag for the print statement! Don't use it with Python 3.
symbols = Symbols(grammar)
pytree.Init(symbols) # for type_repr() pretty printing
transformer.Init(symbols) # for _names and other dicts
if 'PYTHON2' in os.environ:
grammar = pygram.python_grammar
pass
else:
grammar = pygram.python_grammar_no_print_statement
# lib2to3 had a flag for the print statement! Don't use it with Python 3.
del grammar.keywords["print"]
# TODO: Now hook convert to generate Python.asdl?
#
@@ -47,26 +99,91 @@ def main(argv):
#
# Libraries: optparse, etc.
d = driver.Driver(grammar, convert=pytree.convert)
do_glue = False
do_glue = True
if do_glue: # Make it a flag
# Emulating parser.st structures from parsermodule.c.
# They have a totuple() method, which outputs tuples like this.
def py2st(grammar, raw_node):
type, value, context, children = raw_node
# See pytree.Leaf
if context:
_, (lineno, column) = context
else:
lineno = 0 # default in Leaf
column = 0
if children:
return (type,) + tuple(children)
else:
return (type, value, lineno, column)
convert = py2st
else:
convert = pytree.convert
py_path = argv[1]
with open(py_path) as f:
tokens = tokenize.generate_tokens(f.readline)
tree = d.parse_tokens(tokens)
d = driver.Driver(grammar, convert=convert)
action = argv[2]
if action == 'stdlib-parse':
# This is what the compiler/ package was written against.
import parser
py_path = argv[3]
with open(py_path) as f:
st = parser.suite(f.read())
tree = st.totuple()
n = transformer.CountTupleTree(tree)
print('COUNT', n)
printer = transformer.TupleTreePrinter(HostStdlibNames())
printer.Print(tree)
elif action == 'parse':
py_path = argv[3]
with open(py_path) as f:
tokens = tokenize.generate_tokens(f.readline)
tree = d.parse_tokens(tokens, start_symbol=FILE_INPUT)
if isinstance(tree, tuple):
n = transformer.CountTupleTree(tree)
print('COUNT', n)
printer = transformer.TupleTreePrinter(transformer._names)
printer.Print(tree)
else:
tree.PrettyPrint(sys.stdout)
print('\tChildren: %d' % len(tree.children), file=sys.stderr)
elif action == 'compile':
DISPLAY = 0 # debugging?
py_path = argv[3]
out_path = argv[4]
if do_glue:
py_parser = Pgen2PythonParser(d, FILE_INPUT)
printer = transformer.TupleTreePrinter(transformer._names)
tr = transformer.Pgen2Transformer(py_parser, printer)
else:
tr = transformer.Transformer()
tree.PrettyPrint(sys.stdout)
print('\tChildren: %d' % len(tree.children), file=sys.stderr)
return
print(tree)
print(repr(tree))
return
#pycodegen.compileFile(py_path, transformer=tr)
#print(type(tree))
for c in tree.children:
print(repr(c))
print()
# Not file_input instead of single_input? Why is this not a Module?
# Because you
with open(py_path) as f:
contents = f.read()
co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
file_size = os.path.getsize(py_path)
print(co)
with open(out_path, 'wb') as out_f:
pycodegen.WritePyc(co, py_path, out_f)
else:
raise RuntimeError('Invalid action %r' % action)
# Examples of nodes Leaf(type, value):
# Leaf(1, 'def')
@@ -82,5 +199,5 @@ def main(argv):
try:
main(sys.argv)
except RuntimeError as e:
print >>sys.stderr, 'FATAL: %s' % e
print('FATAL: %s' % e, file=sys.stderr)
sys.exit(1)
View
4 opy/pgen2/driver.py
@@ -35,11 +35,11 @@ def __init__(self, grammar, convert=None, logger=None):
self.logger = logger
self.convert = convert
def parse_tokens(self, tokens, debug=False):
def parse_tokens(self, tokens, start_symbol=None, debug=False):
"""Parse a series of tokens and return the syntax tree."""
# XXX Move the prefix computation into a wrapper around tokenize.
p = parse.Parser(self.grammar, self.convert)
p.setup()
p.setup(start=start_symbol)
lineno = 1
column = 0
type = value = start = end = line_text = None
View
2 opy/pgen2/grammar.py
@@ -17,7 +17,7 @@
import pickle
# Local imports
from opy.pgen2 import token, tokenize
from pgen2 import token, tokenize
class Grammar(object):
View
7 opy/pgen2/parse.py
@@ -198,4 +198,9 @@ def pop(self):
node[-1].append(newnode)
else:
self.rootnode = newnode
self.rootnode.used_names = self.used_names
try:
self.rootnode.used_names = self.used_names
except AttributeError:
# Don't need this hack?
pass
View
2 opy/pgen2/pgen.py
@@ -5,7 +5,7 @@
# Pgen imports
#import grammar, token, tokenize
# NOTE: Need these special versions of token/tokenize for BACKQUOTE and such.
from opy.pgen2 import grammar, token, tokenize
from pgen2 import grammar, token, tokenize
class PgenGrammar(grammar.Grammar):
pass
View
4 opy/pgen2/tokenize.py
@@ -33,8 +33,8 @@
from codecs import BOM_UTF8, lookup
#from lib2to3.pgen2.token import *
from opy.pgen2 import token
from opy.pgen2.token import *
from pgen2 import token
from pgen2.token import *
__all__ = [x for x in dir(token) if x[0] != '_'] + ["tokenize",
"generate_tokens", "untokenize"]
View
142 opy/py27.grammar
@@ -0,0 +1,142 @@
# Grammar for Python
# Note: Changing the grammar specified in this file will most likely
# require corresponding changes in the parser module
# (../Modules/parsermodule.c). If you can't make the changes to
# that module yourself, please co-ordinate the required changes
# with someone who can; ask around on python-dev for help. Fred
# Drake <fdrake@acm.org> will probably be listening there.
# NOTE WELL: You should also follow all the steps listed in PEP 306,
# "How to Change Python's Grammar"
# Start symbols for the grammar:
# single_input is a single interactive statement;
# file_input is a module or sequence of commands read from an input file;
# eval_input is the input for the eval() and input() functions.
# NB: compound_stmt in single_input is followed by extra NEWLINE!
single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE
file_input: (NEWLINE | stmt)* ENDMARKER
eval_input: testlist NEWLINE* ENDMARKER
decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE
decorators: decorator+
decorated: decorators (classdef | funcdef)
funcdef: 'def' NAME parameters ':' suite
parameters: '(' [varargslist] ')'
varargslist: ((fpdef ['=' test] ',')*
('*' NAME [',' '**' NAME] | '**' NAME) |
fpdef ['=' test] (',' fpdef ['=' test])* [','])
fpdef: NAME | '(' fplist ')'
fplist: fpdef (',' fpdef)* [',']
stmt: simple_stmt | compound_stmt
simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt |
import_stmt | global_stmt | exec_stmt | assert_stmt)
expr_stmt: testlist (augassign (yield_expr|testlist) |
('=' (yield_expr|testlist))*)
augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
'<<=' | '>>=' | '**=' | '//=')
# For normal assignments, additional restrictions enforced by the interpreter
print_stmt: 'print' ( [ test (',' test)* [','] ] |
'>>' test [ (',' test)+ [','] ] )
del_stmt: 'del' exprlist
pass_stmt: 'pass'
flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt
break_stmt: 'break'
continue_stmt: 'continue'
return_stmt: 'return' [testlist]
yield_stmt: yield_expr
raise_stmt: 'raise' [test [',' test [',' test]]]
import_stmt: import_name | import_from
import_name: 'import' dotted_as_names
import_from: ('from' ('.'* dotted_name | '.'+)
'import' ('*' | '(' import_as_names ')' | import_as_names))
import_as_name: NAME ['as' NAME]
dotted_as_name: dotted_name ['as' NAME]
import_as_names: import_as_name (',' import_as_name)* [',']
dotted_as_names: dotted_as_name (',' dotted_as_name)*
dotted_name: NAME ('.' NAME)*
global_stmt: 'global' NAME (',' NAME)*
exec_stmt: 'exec' expr ['in' test [',' test]]
assert_stmt: 'assert' test [',' test]
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated
if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite]
while_stmt: 'while' test ':' suite ['else' ':' suite]
for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite]
try_stmt: ('try' ':' suite
((except_clause ':' suite)+
['else' ':' suite]
['finally' ':' suite] |
'finally' ':' suite))
with_stmt: 'with' with_item (',' with_item)* ':' suite
with_item: test ['as' expr]
# NB compile.c makes sure that the default except clause is last
except_clause: 'except' [test [('as' | ',') test]]
suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT
# Backward compatibility cruft to support:
# [ x for x in lambda: True, lambda: False if x() ]
# even while also allowing:
# lambda x: 5 if x else 2
# (But not a mix of the two)
testlist_safe: old_test [(',' old_test)+ [',']]
old_test: or_test | old_lambdef
old_lambdef: 'lambda' [varargslist] ':' old_test
test: or_test ['if' or_test 'else' test] | lambdef
or_test: and_test ('or' and_test)*
and_test: not_test ('and' not_test)*
not_test: 'not' not_test | comparison
comparison: expr (comp_op expr)*
comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
expr: xor_expr ('|' xor_expr)*
xor_expr: and_expr ('^' and_expr)*
and_expr: shift_expr ('&' shift_expr)*
shift_expr: arith_expr (('<<'|'>>') arith_expr)*
arith_expr: term (('+'|'-') term)*
term: factor (('*'|'/'|'%'|'//') factor)*
factor: ('+'|'-'|'~') factor | power
power: atom trailer* ['**' factor]
atom: ('(' [yield_expr|testlist_comp] ')' |
'[' [listmaker] ']' |
'{' [dictorsetmaker] '}' |
'`' testlist1 '`' |
NAME | NUMBER | STRING+)
listmaker: test ( list_for | (',' test)* [','] )
testlist_comp: test ( comp_for | (',' test)* [','] )
lambdef: 'lambda' [varargslist] ':' test
trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
subscriptlist: subscript (',' subscript)* [',']
subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: expr (',' expr)* [',']
testlist: test (',' test)* [',']
dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
(test (comp_for | (',' test)* [','])) )
classdef: 'class' NAME ['(' [testlist] ')'] ':' suite
arglist: (argument ',')* (argument [',']
|'*' test (',' argument)* [',' '**' test]
|'**' test)
# The reason that keywords are test nodes instead of NAME is that using NAME
# results in an ambiguity. ast.c makes sure it's a NAME.
argument: test [comp_for] | test '=' test
list_iter: list_for | list_if
list_for: 'for' exprlist 'in' testlist_safe [list_iter]
list_if: 'if' old_test [list_iter]
comp_iter: comp_for | comp_if
comp_for: 'for' exprlist 'in' or_test [comp_iter]
comp_if: 'if' old_test [comp_iter]
testlist1: test (',' test)*
# not used in grammar, but may appear in "node" passed from Parser to Compiler
encoding_decl: NAME
yield_expr: 'yield' [testlist]
View
40 opy/pygram.py

This file was deleted.

Oops, something went wrong.
View
22 opy/pytree.py
@@ -19,15 +19,21 @@
HUGE = 0x7FFFFFFF # maximum repeat count, default max
_type_reprs = {}
def type_repr(type_num):
global _type_reprs
if not _type_reprs:
from .pygram import python_symbols
# printing tokens is possible but not as useful
# from .pgen2 import token // token.__dict__.items():
for name, val in python_symbols.__dict__.items():
if type(val) == int: _type_reprs[val] = name
return _type_reprs.setdefault(type_num, type_num)
return _type_reprs[type_num]
def Init(python_symbols):
# printing tokens is possible but not as useful
# from .pgen2 import token // token.__dict__.items():
# Fill in the dict.
global _type_reprs
for name, val in python_symbols.__dict__.items():
if type(val) == int:
_type_reprs[val] = name
class Base(object):
View
95 opy/run.sh
@@ -10,10 +10,84 @@ set -o errexit
readonly PY=~/src/Python-3.6.1
readonly DIFF=${DIFF:-diff -u}
_parse-one() {
PYTHONPATH=. ./parse.py 2to3.grammar parse "$@"
}
_stdlib-parse-one() {
PYTHONPATH=. ./parse.py 2to3.grammar stdlib-parse "$@"
}
_compile-one() {
# The production testlist_starexpr is unhandled in the compiler package.
# Python 2.7 doesn't have it.
#local g=2to3.grammar
local g=py27.grammar
PYTHONPATH=. ./parse.py $g compile "$@"
}
parse-test() {
_parse-one testdata/hello_py3.py
echo ---
PYTHON2=1 _parse-one testdata/hello_py2.py
}
stdlib-parse-test() {
_stdlib-parse-one testdata/hello_py3.py
echo ---
PYTHON2=1 _stdlib-parse-one testdata/hello_py2.py
}
compile-gold() {
pushd testdata
python3 -c 'import hello_py3'
ls -l __pycache__
xxd __pycache__/hello_py3.cpython-34.pyc
popd
}
_compile-and-run() {
local path=$1
local basename=$(basename $path .py)
mkdir -p _tmp
# Doesn't work why? Because it was written for Python 2.7?
local out=_tmp/${basename}.pyc
_compile-one $path $out
ls -l $out
xxd $out
# Crap, it doesn't work!
python3 $out
return
echo ---
# This doesn't work because compiler does 'import parser', which is the
# Python 3 paresr now!
PYTHON2=1 _compile-one testdata/hello_py2.py
}
compile-hello() {
_compile-and-run testdata/hello_py3.py
}
compile-self() {
_compile-and-run ./parse.py
}
old-compile-test() {
PYTHONPATH=. tools/compile.py testdata/hello_py3.py
}
# 2to3.grammar is from Python-3.6.1/ Lib/lib2to3/Grammar.txt
parse-with-pgen2() {
set +o errexit
for py in "$@"; do
PYTHONPATH=.. ./parse.py $py >/dev/null #2>&1
_parse-one $py >/dev/null #2>&1
echo $? $py
done
}
@@ -31,9 +105,13 @@ parse-pycompiler2() {
# After lib2to3
parse-pycompiler() {
# parse print statement
parse-with-pgen2 compiler/*.py
parse-with-pgen2 compiler/*.py tools/*.py
}
#
# File Management
#
clear-tokens() {
rm token.py tokenize.py
rm -rf --verbose __pycache ../__pycache__
@@ -55,8 +133,12 @@ copy-lib2to3() {
cp -v $PY/Parser/Python.asdl .
# For comparison
#cp -v $PY/Grammar/Grammar .
}
# For compatibility with the compiler package.
# I kind of want the static type syntax though?
copy-old-grammar() {
cp -v $PY27/Grammar/Grammar py27.grammar
}
copy-pycompiler() {
@@ -80,8 +162,11 @@ test-pgen-parse() {
PYTHONPATH=.. ./pgen_parse.py Grammar.txt pgen_parse.py
}
readonly PY27=~/src/Python-2.7.6
compare-grammar() {
$DIFF Grammar Grammar.txt
# The compiler package was written for a different grammar!
$DIFF $PY27/Grammar/Grammar 2to3.grammar
}
# pgen2 has BACKQUOTE = 25. No main.
View
1 opy/testdata/hello_py2.py
@@ -0,0 +1 @@
print "hi"
View
1 opy/testdata/hello_py3.py
@@ -0,0 +1 @@
print("hi")