Build opy.ovm with the grammar pickle.

It can successfully parse with 'opyc parse', but this exposed a bug in
'opyc compile'.  transformer.py uses eval(), which doesn't work because
the parser isn't present in OVM!

build/actions.sh

@@ -200,8 +200,8 @@ clean-pyc() {
# we should put it in a different directory?
clean-repo() {
  clean-source-tarball-build
  rm -r -f _build _release
  rm -f _bin/hello.*
  rm -r -f -v _build _release
  rm -f -v _bin/hello.* _bin/opy.*
  clean-pyc
}

build/opy.mk

@@ -24,14 +24,19 @@ _build/opy/py-to-compile.txt: _build/detected-config.sh build/app_deps.py
	  $(ACTIONS_SH) py-to-compile $(REPO_ROOT) bin.opy_ > $@


# TODO: oil-version can be like this too.
GRAMMAR = _build/opy/py27.grammar.pickle

OPY_BYTECODE_DEPS := \
	_build/release-date.txt \
	build/opy-manifest.txt
	build/opy-manifest.txt \
	$(GRAMMAR)

# NOTE: runpy deps are included in opy-app-deps.txt.
_build/opy/bytecode-opy-manifest.txt: $(OPY_BYTECODE_DEPS) \
                         _build/opy/opy-app-deps.txt
_build/opy/bytecode-opy-manifest.txt: \
	$(OPY_BYTECODE_DEPS) _build/opy/opy-app-deps.txt
	{ echo '_build/release-date.txt release-date.txt'; \
	  echo $(GRAMMAR) $(GRAMMAR); \
	  cat build/opy-manifest.txt \
	      _build/opy/opy-app-deps.txt; \
	  $(ACTIONS_SH) pyc-version-manifest $@; \

opy/build.sh

@@ -71,7 +71,7 @@ compile-manifest() {
    local dest=$dest_dir/$rel_dest_path
    mkdir -p $(dirname $dest)
    log "     $full_src_path"
    _compile-one $full_src_path $dest
    bin/opyc compile $full_src_path $dest

    local rel_py_path=${rel_dest_path%.pyc}.py   # .pyc -> py

opy/common.sh

@@ -25,27 +25,6 @@ opy_() {
  PYTHONPATH=$THIS_DIR $THIS_DIR/../bin/opy_.py "$@"
}

opyg() {
  opy_ -g $THIS_DIR/$GRAMMAR -- "$@"
}

# The old compile path
_compile-one() {
  # The production testlist_starexpr is unhandled in the compiler package.
  # Python 2.7 doesn't have it.
  #local g=2to3.grammar 
  local g=py27.grammar
  # pgen2 + old compiler/
  #PYTHONPATH=. ./opy_main.py $g old-compile "$@"

  # opy is pgen2 + compiler2
  opyg compile "$@"
}

_compile2-one() {
  opyg compile2 "$@"
}

_stdlib-compile-one() {
  # Run it from source, so we can patch.  Bug still appears.

opy/opy_main.py

@@ -145,23 +145,35 @@ def OpyCommandMain(argv):
  # TODO: Use core/args.
  opts, argv = Options().parse_args(argv)

  loader = util.GetResourceLoader()
  f = loader.open(PICKLE_REL_PATH)
  gr = grammar.Grammar()
  gr.load(f)
  f.close()

  # In Python 2 code, always use from __future__ import print_function.
  try:
    del gr.keywords["print"]
  except KeyError:
    pass
    action = argv[0]
  except IndexError:
    raise args.UsageError('opy: Missing required subcommand.')

  if action in ('parse', 'compile'):
    loader = util.GetResourceLoader()
    f = loader.open(PICKLE_REL_PATH)
    gr = grammar.Grammar()
    gr.load(f)
    f.close()

    # In Python 2 code, always use from __future__ import print_function.
    try:
      del gr.keywords["print"]
    except KeyError:
      pass

  FILE_INPUT = gr.symbol2number['file_input']
    FILE_INPUT = gr.symbol2number['file_input']

  symbols = Symbols(gr)
  pytree.Init(symbols)  # for type_repr() pretty printing
  transformer.Init(symbols)  # for _names and other dicts
    symbols = Symbols(gr)
    pytree.Init(symbols)  # for type_repr() pretty printing
    transformer.Init(symbols)  # for _names and other dicts
  else:
    # e.g. pgen2 doesn't use any of these.  Maybe we should make a different
    # tool.
    gr = None
    FILE_INPUT = None
    symbols = None

  #do_glue = False
  do_glue = True
@@ -188,11 +200,6 @@ def py2st(gr, raw_node):

  dr = driver.Driver(gr, convert=convert)

  try:
    action = argv[0]
  except IndexError:
    raise args.UsageError('opy: Missing required subcommand.')

  if action == 'pgen2':
    grammar_path = argv[1]
    pickle_path = argv[2]

test/opy.sh

@@ -9,7 +9,9 @@ set -o errexit

source test/common.sh

readonly OPYC=${OPYC:-bin/opyc}
# Can't be readonly because we override it below?  Gah.
#readonly OPYC=${OPYC:-bin/opyc}
OPYC=${OPYC:-bin/opyc}

readonly TMP_DIR=_tmp/opy-test
mkdir -p $TMP_DIR
@@ -86,4 +88,9 @@ all-passing() {
  run-all "${PASSING[@]}"
}

# Use the release binary
run-for-release() {
  OPYC=_bin/opyc $0 run-all "${PASSING[@]}"
}

"$@"