View
73 core/lexer_gen.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
"""
lex_gen.py
"""
@@ -22,31 +23,31 @@ def PrintTree(re_tree, depth=2):
sys.stdout.write(name)
sys.stdout.write(' ')
if name == 'in': # character class
print '{'
print('{')
PrintTree(arg, depth=depth+1)
sys.stdout.write(depth * '\t')
print '}'
print('}')
elif name == 'max_repeat': # repetition
min_, max_, children = arg
# min = 0 means *, min = 1 means +
assert min_ in (0, 1), min_
print min_, max_, '{'
print(min_, max_, '{')
PrintTree(children, depth=depth+1)
sys.stdout.write(depth * '\t')
print
print()
elif name == 'negate': # Oh this is a ^. It doesn't form a node.
assert arg is None
print
print()
elif name == 'literal': # Quote \ and " in re2c syntax
print repr(chr(arg))
print(repr(chr(arg)))
elif name == 'not_literal': # ditto
print repr(chr(arg))
print(repr(chr(arg)))
elif name == 'range': # ascii range
begin, end = arg
print repr(chr(begin)), repr(chr(end))
print(repr(chr(begin)), repr(chr(end)))
elif name == 'any': # This is the '.' character
assert arg is None
print
print()
else:
raise AssertionError(name)
@@ -55,9 +56,9 @@ def PrintTree(re_tree, depth=2):
def PrintRegex(pat):
re_tree = sre_parse.parse(pat)
print '\t\t['
print('\t\t[')
PrintTree(re_tree)
print '\t\t]'
print('\t\t]')
# ^ means negation, - means range
@@ -194,7 +195,7 @@ def TranslateLexer(lexer_def):
# Since we reference this function in exactly one translation unit --
# fastlex.c, the difference is moot, and we just satisfy the compiler.
print r"""
print(r"""
/* Common stuff */
/*!re2c
@@ -212,34 +213,34 @@ def TranslateLexer(lexer_def):
unsigned char* YYMARKER; /* why do we need this? */
switch (lex_mode) {
"""
""")
# TODO: Should be ordered by most common? Or will profile-directed feedback
# help?
for state, pat_list in lexer_def.iteritems():
# HACK: strip off '_e'
prefix = state.__class__.__name__[:-2]
print ' case %s__%s:' % (prefix, state.name)
print ' for (;;) {'
print ' /*!re2c'
print(' case %s__%s:' % (prefix, state.name))
print(' for (;;) {')
print(' /*!re2c')
for is_regex, pat, token_id in pat_list:
if is_regex:
re2_pat = TranslateRegex(pat)
else:
re2_pat = TranslateConstant(pat)
id_name = meta.IdName(token_id)
print ' %-30s { *id = id__%s; break; }' % (re2_pat, id_name)
print(' %-30s { *id = id__%s; break; }' % (re2_pat, id_name))
# EARLY RETURN: Do NOT advance past the NUL terminator.
print ' %-30s { *id = id__Eol_Tok; *end_pos = start_pos; return; }' % \
r'"\x00"'
print(' %-30s { *id = id__Eol_Tok; *end_pos = start_pos; return; }' % \
r'"\x00"')
print ' */'
print ' }'
print ' break;'
print
print(' */')
print(' }')
print(' break;')
print()
# This is literal code without generation:
"""
@@ -275,14 +276,14 @@ def TranslateLexer(lexer_def):
break;
"""
print """\
print("""\
default:
assert(0);
}
*end_pos = p - line; /* relative */
}
"""
""")
# note: use YYCURSOR and YYLIMIT
@@ -299,44 +300,44 @@ def main(argv):
elif action == 'print-all':
# Top level is a switch statement.
for state, pat_list in lex.LEXER_DEF.iteritems():
print state
print(state)
# This level is re2c patterns.
for is_regex, pat, token_id in pat_list:
print '\t%r -> %r' % (pat, token_id)
print('\t%r -> %r' % (pat, token_id))
if is_regex:
#print re_tree
out_pat = TranslateRegex(pat)
#print out_pat
print
print()
elif action == 'print-regex':
unique = set()
num_regexes = 0
for state, pat_list in lex.LEXER_DEF.iteritems():
print state
print(state)
# This level is re2c patterns.
for is_regex, pat, token_id in pat_list:
#print '\t%r -> %r' % (pat, token_id)
if is_regex:
print '\t' + pat
print '\t' + TranslateRegex(pat)
print
print('\t' + pat)
print('\t' + TranslateRegex(pat))
print()
#PrintRegex(pat)
num_regexes += 1
unique.add(pat)
else:
print '\t' + TranslateConstant(pat)
print('\t' + TranslateConstant(pat))
print
print()
print 'Printed %d regexes (%d unique)' % (num_regexes, len(unique))
print('Printed %d regexes (%d unique)' % (num_regexes, len(unique)))
if __name__ == '__main__':
try:
main(sys.argv)
except RuntimeError as e:
print >>sys.stderr, 'FATAL: %s' % e
print('FATAL: %s' % e, file=sys.stderr)
sys.exit(1)
View
9 core/lexer_gen_test.py
@@ -1,4 +1,5 @@
#!/usr/bin/python -S
from __future__ import print_function
"""
lexer_gen_test.py: Tests for lexer_gen.py
"""
@@ -37,12 +38,12 @@ def testTranslateRegex(self):
]
for py, expected in PAIRS:
#self.assertEqual(expected, lexer_gen.TranslateRegex(py))
print '---', py
print('---', py)
actual = lexer_gen.TranslateRegex(py)
print repr(actual)
print(repr(actual))
self.assertEqual(expected, actual)
print
print
print()
print()
if __name__ == '__main__':
View
19 core/libstr_test.py
@@ -1,4 +1,5 @@
#!/usr/bin/python -S
from __future__ import print_function
"""
libstr_test.py: Tests for libstr.py
"""
@@ -11,7 +12,7 @@
class LibStrTest(unittest.TestCase):
def testUnarySuffixOpDemo(self):
print libstr
print(libstr)
s = 'abcd'
n = len(s)
@@ -21,23 +22,23 @@ def testUnarySuffixOpDemo(self):
print('## shortest prefix')
for i in xrange(1, n+1):
print '%d test %06r return %06r' % (i, s[:i], s[i:])
print
print('%d test %06r return %06r' % (i, s[:i], s[i:]))
print()
print('# longest prefix')
for i in xrange(n, 0, -1):
print '%d test %06r return %06r' % (i, s[:i], s[i:])
print
print('%d test %06r return %06r' % (i, s[:i], s[i:]))
print()
print('% shortest suffix')
for i in xrange(n-1, -1, -1):
print '%d test %06r return %06r' % (i, s[i:], s[:i])
print
print('%d test %06r return %06r' % (i, s[i:], s[:i]))
print()
print('%% longest suffix')
for i in xrange(0, n):
print '%d test %06r return %06r' % (i, s[i:], s[:i])
print
print('%d test %06r return %06r' % (i, s[i:], s[:i]))
print()
if __name__ == '__main__':
View
3 core/word_eval_test.py
@@ -5,6 +5,7 @@
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
from __future__ import print_function
"""
word_eval_test.py: Tests for word_eval.py
"""
@@ -17,7 +18,7 @@
class WordEvalTest(unittest.TestCase):
def testWordEval(self):
print word_eval
print(word_eval)
if __name__ == '__main__':
View
3 core/word_test.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
"""
word_test.py: Tests for word.py
"""
@@ -11,7 +12,7 @@
class WordTest(unittest.TestCase):
def testFoo(self):
print word
print(word)
if __name__ == '__main__':
View
18 opy/README.md
@@ -0,0 +1,18 @@
OPy Compiler
============
Getting started / smoke test:
./build.sh grammar
./run.sh parse-test
./run.sh compile-hello2 # prints hello world
./run.sh compile-hello3 # no __future__ print_function
Compiling Oil:
./smoke.sh compile-osh-tree
TODO:
Move important stuff to build.sh. smoke.sh doesn't make sense.
View
1 opy/build.sh
@@ -10,6 +10,7 @@ set -o errexit
source common.sh
grammar() {
mkdir -p _tmp
opy_ pgen2 py27.grammar $GRAMMAR
}
View
5 opy/common.sh
@@ -10,6 +10,11 @@ set -o errexit
readonly THIS_DIR=$(cd $(dirname $0) && pwd)
readonly GRAMMAR=$THIS_DIR/_tmp/py27.grammar.pickle
die() {
echo "FATAL: $@" 1>&2
exit 1
}
opy_() {
PYTHONPATH=$THIS_DIR $THIS_DIR/../bin/opy_.py "$@"
}
View
15 opy/fix.sh
@@ -0,0 +1,15 @@
#!/bin/bash
#
# Usage:
# ./fix.sh <function name>
set -o nounset
set -o pipefail
set -o errexit
# -w to write it back
print() {
2to3 --fix print "$@"
}
"$@"
View
22 opy/run.sh
@@ -12,18 +12,13 @@ source compare.sh
readonly PY=$PY36
die() {
echo "FATAL: $@" 1>&2
exit 1
}
_parse-one() {
#PYTHONPATH=. ./opy_main.py 2to3.grammar parse "$@"
opyg parse "$@"
}
parse-test() {
_parse-one testdata/hello_py3.py
_parse-one testdata/hello_py3.py # Python 3 print syntax
echo ---
_parse-one testdata/hello_py2.py
}
@@ -39,17 +34,6 @@ stdlib-parse-test() {
_stdlib-parse-one testdata/hello_py2.py
}
# Generate .pyc using the Python interpreter.
compile-gold() {
pushd testdata
python3 -c 'import hello_py3'
ls -l __pycache__
xxd __pycache__/hello_py3.cpython-34.pyc
popd
}
_compile-and-run() {
local path=$1
local basename=$(basename $path .py)
@@ -120,7 +104,6 @@ stdlib-determinism-loop() {
determinism-loop _stdlib-compile-one
}
# We want to fix the bug here. Hm not able to hit it?
compile2-determinism() {
mkdir -p _tmp/det
@@ -194,7 +177,7 @@ unit-osh() {
popd
}
# Combinatios of {ccompile, compiler2} x {cpython, byterun}
# Combinations of {ccompile, compiler2} x {cpython, byterun}
compile-run-one() {
local compiler=${1:-ccompile} # or compile2
local vm=${2:-byterun} # or cpython
@@ -257,6 +240,7 @@ parse-with-pgen2() {
done
}
# This fails due to some files not using __future__ print_function.
parse-oil() {
parse-with-pgen2 *.py ../*.py ../osh/*.py ../core/*.py ../asdl/*.py
}
View
6 opy/smoke.sh
@@ -81,10 +81,16 @@ _fill-osh-tree() {
ln -v -s -f $PWD/../core/libc.so $dir/core
}
# Hm should we support Python 2 print? It will be useful for converting
# many old files. Otherwise, we need a quick '2to3' alias that only does
# the conversion.
compile-osh-tree() {
local src=$(cd .. && echo $PWD)
local files=( $(find $src \
-name _tmp -a -prune -o \
-name _chroot -a -prune -o \
-name _devbuild -a -prune -o \
-name Python-2.7.13 -a -prune -o \
-name opy -a -prune -o \
-name tests -a -prune -o \
-name '*.py' -a -printf '%P\n') )
View
3 osh/lex_test.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
"""
lex_test.py: Tests for lex.py
"""
@@ -24,7 +25,7 @@ def _InitLexer(s):
class AsdlTest(unittest.TestCase):
def testLexMode(self):
print lex_mode_e.DQ
print(lex_mode_e.DQ)
CMD = """\
View
5 spec/bin/show_fd_table.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
"""
show_fd_table.py -- Uses Linux-specific proc interface
"""
@@ -14,9 +15,9 @@ def main(argv):
try:
connected_to = os.readlink(path)
except OSError as e:
print fd, str(e)
print('%s %s' % (fd, e))
else:
print fd, connected_to
print('%s %s' % (fd, connected_to))
if __name__ == '__main__':
View
11 test/wild_report.py
@@ -1,4 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
"""
wild_report.py
"""
@@ -321,7 +322,7 @@ def MakeHtmlGroup(title_str, body_str):
def log(msg, *args):
if msg:
msg = msg % args
print >>sys.stderr, msg
print(msg, file=sys.stderr)
class DirNode:
@@ -402,9 +403,9 @@ def DebugPrint(node, indent=0):
ind = indent * ' '
#print('FILES', node.files.keys())
for name in node.files:
print '%s%s - %s' % (ind, name, node.files[name])
print('%s%s - %s' % (ind, name, node.files[name]))
for name, child in node.dirs.iteritems():
print '%s%s/ - %s' % (ind, name, child.subtree_stats)
print('%s%s/ - %s' % (ind, name, child.subtree_stats))
DebugPrint(child, indent=indent+1)
@@ -493,7 +494,7 @@ def WriteHtmlFiles(node, out_dir, rel_path='', base_url=''):
st['lines_per_sec'] = '%.1f' % lines_per_sec
except KeyError:
# This usually there were ZERO files.
print >>sys.stderr, node, st, repr(rel_path)
print(node, st, repr(rel_path), file=sys.stderr)
raise
data = {
@@ -597,5 +598,5 @@ def _ReadTaskFile(path):
try:
main(sys.argv)
except RuntimeError as e:
print >>sys.stderr, 'FATAL: %s' % e
print('FATAL: %s' % e, file=sys.stderr)
sys.exit(1)