The 'opyc parse' and 'opyc compile' entry points now work.

Instead of the -g flag, we build the grammar pickle with a Makefile rule
and place it in a canonical location.  Use Oil's app bundle resource
loader too.

build/opy.mk

@@ -9,6 +9,9 @@

-include _build/opy/ovm.d

_build/opy/py27.grammar.pickle:
	bin/opyc pgen2 opy/py27.grammar $@

_build/opy/main_name.c:
	$(ACTIONS_SH) main-name bin.opy_ opy.ovm > $@

opy/opy_main.py

@@ -31,6 +31,7 @@
from .util_opy import log

from core import args
from core import util


# From lib2to3/pygram.py.  This takes the place of the 'symbol' module.
@@ -63,13 +64,6 @@ def HostStdlibNames():
  return names


def LoadGrammar(pickle_path):
  """Load the grammar (maybe from a pickle)."""
  g = grammar.Grammar()
  g.load(pickle_path)  # pickle.load()
  return g


def WriteGrammar(grammar_path, pickle_path):
  log("Generating grammar tables from %s", grammar_path)
  g = pgen.generate_grammar(grammar_path)
@@ -139,34 +133,35 @@ def Options():
  p.add_option(
      '-c', dest='command', default=None,
      help='Python command to run')
  p.add_option(
      '-g', dest='grammar', default=None,
      help='Grammar pickle file to use for parsing')
  return p


# Made by the Makefile.
PICKLE_REL_PATH = '_build/opy/py27.grammar.pickle'

def OpyCommandMain(argv):
  """Dispatch to the right action."""

  # TODO: Use core/args.
  opts, argv = Options().parse_args(argv)

  if opts.grammar:
    gr = LoadGrammar(opts.grammar)
    # In Python 2 code, always use from __future__ import print_function.
    try:
      del gr.keywords["print"]
    except KeyError:
      pass
  loader = util.GetResourceLoader()
  f = loader.open(PICKLE_REL_PATH)
  gr = grammar.Grammar()
  gr.load(f)
  f.close()

    FILE_INPUT = gr.symbol2number['file_input']
  # In Python 2 code, always use from __future__ import print_function.
  try:
    del gr.keywords["print"]
  except KeyError:
    pass

    symbols = Symbols(gr)
    pytree.Init(symbols)  # for type_repr() pretty printing
    transformer.Init(symbols)  # for _names and other dicts
  else:
    gr = None
    FILE_INPUT = None
    symbols = None
  FILE_INPUT = gr.symbol2number['file_input']

  symbols = Symbols(gr)
  pytree.Init(symbols)  # for type_repr() pretty printing
  transformer.Init(symbols)  # for _names and other dicts

  #do_glue = False
  do_glue = True
@@ -234,7 +229,9 @@ def py2st(gr, raw_node):
      tree.PrettyPrint(sys.stdout)
      log('\tChildren: %d' % len(tree.children), file=sys.stderr)

  elif action == 'old-compile':
  elif action == 'compile':
    # 'opy compile' is pgen2 + compiler2

    py_path = argv[1]
    out_path = argv[2]

@@ -245,8 +242,8 @@ def py2st(gr, raw_node):
    else:
      tr = transformer.Transformer()

    f = open(py_path)
    contents = f.read()
    with open(py_path) as f:
      contents = f.read()
    co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
    log("Code length: %d", len(co.co_code))

@@ -256,10 +253,8 @@ def py2st(gr, raw_node):
      out_f.write(h)
      marshal.dump(co, out_f)

  elif action == 'compile':
    # 'opy compile' is pgen2 + compiler2
    # TODO: import compiler2
    #raise NotImplementedError
  # NOTE: Unused
  elif action == 'old-compile':
    py_path = argv[1]
    out_path = argv[2]

@@ -270,8 +265,8 @@ def py2st(gr, raw_node):
    else:
      tr = transformer.Transformer()

    with open(py_path) as f:
      contents = f.read()
    f = open(py_path)
    contents = f.read()
    co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
    log("Code length: %d", len(co.co_code))

@@ -281,6 +276,7 @@ def py2st(gr, raw_node):
      out_f.write(h)
      marshal.dump(co, out_f)

  # TODO: Not used
  elif action == 'compile2':
    in_path = argv[1]
    out_path = argv[2]

opy/pgen2/grammar.py

@@ -119,11 +119,8 @@ def dump(self, filename):
              s += marshal.dumps(self.start)
              print('Marshalled bytes: %d' % len(s))

    def load(self, filename):
        """Load the grammar tables from a pickle file."""
        with open(filename, "rb") as f:
            d = pickle.load(f)
            #d = marshal.load(f)
    def load(self, f):
        d = pickle.load(f)
        self.__dict__.update(d)

    def copy(self):

test/opy.sh

@@ -9,6 +9,12 @@ set -o errexit

source test/common.sh

readonly OPYC=${OPYC:-bin/opyc}

readonly TMP_DIR=_tmp/opy-test
mkdir -p $TMP_DIR


usage() {
  set +o errexit

@@ -45,22 +51,39 @@ usage() {
  #test $? -eq 0 || fail
}

parse() {
  cat >$TMP_DIR/hello.py <<EOF
print(1+2)
EOF
  $OPYC parse $TMP_DIR/hello.py
}

compile() {
  cat >$TMP_DIR/loop.py <<EOF
for i in xrange(4):
  print(i*i)
EOF

  $OPYC compile $TMP_DIR/loop.py $TMP_DIR/loop.opyc

  # We can run it with CPython now, but later we won't able to.
  python $TMP_DIR/loop.opyc > out.txt
  diff out.txt - <<EOF || fail
0
1
4
9
EOF
}

readonly -a PASSING=(
  usage
  parse
  compile
)

# TODO: Consolidate this

all-passing() {
  for t in "${PASSING[@]}"; do
    # fail calls 'exit 1'
    $t
    echo "OK  $t"
  done

  echo
  echo "All $0 tests passed."
  run-all "${PASSING[@]}"
}


"$@"