Broke runtime dependency on asdl/front_end.py!

- Used pickle module to serialize the graph of _RuntimeType objects.
  We're using the pure Python pickle so we don't depend on more C code.
- The generated _devbuild/gen/*_asdl.py now loads pickles.
- Moved both Python and C code generation in to osh/asdl_gen.py.
- The build/dev.sh shell script has to use a BOOTSTRAP_LEVEL hack to get
  around a circular dependency involving Id.

Also:

- Simplify line counts for ASDL.

asdl/asdl_.py

@@ -24,6 +24,10 @@
import cStringIO


# TODO: There should be SimpleSumType(_SumType) and CompoundSumType(_SumType)
# That can be determined at compile time with this function.  is_simple()
# should move to front_end.py.

# PATCH: Moved this function from asdl_c.py.
def is_simple(sum):
  """Return True if a sum is a simple.

asdl/asdl_demo.py

@@ -50,15 +50,18 @@ def main(argv):
      import marshal
      import cPickle
      print(dir(marshal))
      with open('out.marshal', 'w') as f:
      out_path = schema_path + '.pickle'
      with open(out_path, 'w') as f:
        #marshal.dump(type_lookup, f)
        cPickle.dump(type_lookup, f)
        # Version 2 is the highest protocol for Python 2.7.
        cPickle.dump(type_lookup.runtime_type_lookup, f, protocol=2)

      print('runtime_type_lookup:')
      for name, desc in type_lookup.runtime_type_lookup.items():
        print(name)
        print(desc)
      print()
      print('Wrote %s' % out_path)

  elif action == 'arith-encode':  # oheap encoding
    expr = argv[2]

asdl/front_end.py

@@ -1,4 +1,5 @@
#!/usr/bin/python
from __future__ import print_function
"""
front_end.py: Lexer and parser for the ASDL schema language.
"""
@@ -281,13 +282,11 @@ def _AppendFields(field_ast_nodes, type_lookup, out):
    out.append((field.name, runtime_type))


def _MakeReflection(module, app_types=None):
def _MakeReflection(module, app_types):
  # Types that fields are declared with: int, id, word_part, etc.
  # Fields are NOT declared with Constructor names.
  type_lookup  = dict(asdl.BUILTIN_TYPES)

  if app_types is not None:
    type_lookup.update(app_types)
  type_lookup.update(app_types)

  # NOTE: We need two passes.  Types can be mutually recurisve.  See
  # asdl/arith.asdl.
@@ -328,7 +327,11 @@ def LoadSchema(f, app_types):
  
  Used for code gen and metaprogramming.

  TODO: Break dependency on the parser.  Used for type_lookup.
  Note: I think app_types is only used for dynamic type checking in
  asdl/py_meta.py.  I guess it could be used for pretty-printing, but that uses
  the actual value and not the type.

  TODO: We should change pretty-printing to also verify the types!
  """
  p = ASDLParser()
  schema_ast = p.parse(f)

asdl/gen_cpp.py

@@ -32,8 +32,6 @@
from asdl import front_end
from asdl import visitor

from osh.meta import Id

class ChainOfVisitors:
  def __init__(self, *visitors):
    self.visitors = visitors
@@ -310,6 +308,11 @@ def main(argv):
  if action == 'cpp':
    schema_path = argv[2]

    # NOTE: This import can't be at the top level osh/asdl_gen.py depends on
    # this gen_cpp.py module.  We should move all the main() functions out of
    # asdl/ and into command line tools.

    from osh.meta import Id
    app_types = {'id': asdl.UserType(Id)}
    with open(schema_path) as input_f:
      module, type_lookup = front_end.LoadSchema(input_f, app_types)

asdl/gen_python.py

@@ -1,4 +1,5 @@
#!/usr/bin/env python
from __future__ import print_function
"""
gen_python.py

@@ -105,33 +106,3 @@ def VisitProduct(self, product, name, depth):

  def EmitFooter(self):
    pass


def main(argv):

  schema_path = argv[1]
  type_lookup_import = argv[2]

  p = front_end.ASDLParser()
  with open(schema_path) as input_f:
    module = p.parse(input_f)

  f = sys.stdout

  f.write("""\
from asdl import const  # For const.NO_INTEGER
from asdl import py_meta
%s

""" % type_lookup_import)

  v = GenClassesVisitor(f)
  v.VisitModule(module)


if __name__ == '__main__':
  try:
    main(sys.argv)
  except RuntimeError as e:
    print >>sys.stderr, 'FATAL: %s' % e
    sys.exit(1)

asdl/py_meta.py

@@ -307,12 +307,3 @@ def MakeTypes(module, root, type_lookup):

    else:
      raise AssertionError(typ)


def AssignTypes(src_module, dest_module):
  """For generated code."""
  for name in dir(src_module):
    if not name.startswith('__'):
      v = getattr(src_module, name)
      setattr(dest_module, name, v)

asdl/run.sh

@@ -61,6 +61,11 @@ py-cpp() {
  asdl-cpp $schema _tmp/$(basename $schema).h
}

inspect-pickle() {
  # python2 doesn't have this?
  python3 -m pickletools asdl/arith.asdl.pickle
}

gen-python() {
  local schema=${1:-asdl/arith.asdl}
  asdl/gen_python.py $schema

build/codegen.sh

@@ -62,7 +62,7 @@ extract-clang() {
}

types-gen() {
  PYTHONPATH=. osh/ast_gen.py "$@" > _devbuild/gen/osh-types.h
  PYTHONPATH=. osh/asdl_gen.py c osh/types.asdl "$@" > _devbuild/gen/osh-types.h
}

id-gen() {

build/dev.sh

@@ -68,25 +68,34 @@ gen-help() {
  build/doc.sh osh-quick-ref
}

gen-types-asdl() {
  local out=_devbuild/gen/types_asdl.py
  local import='from osh.meta import TYPES_TYPE_LOOKUP as TYPE_LOOKUP'
  PYTHONPATH=. asdl/gen_python.py osh/types.asdl "$import" > $out
# Helper
gen-asdl-py-pickle() {
  local asdl_path=$1  # e.g. osh/osh.asdl

  local name=$(basename $asdl_path .asdl)

  local tmp=_tmp/${name}_asdl.py
  local out=_devbuild/gen/${name}_asdl.py

  PYTHONPATH=. osh/asdl_gen.py py $asdl_path _devbuild/${name}_asdl.pickle > $tmp
  
  # BUG: MUST BE DONE ATOMICALLY ATOMIC; otherwise the Python interpreter can
  # import an empty file!
  mv -v $tmp $out

  echo "Wrote $out"
}

gen-types-asdl() {
  gen-asdl-py-pickle osh/types.asdl
}

gen-osh-asdl() {
  local out=_devbuild/gen/osh_asdl.py
  local import='from osh.meta import OSH_TYPE_LOOKUP as TYPE_LOOKUP'
  PYTHONPATH=. asdl/gen_python.py osh/osh.asdl "$import" > $out
  echo "Wrote $out"
  gen-asdl-py-pickle osh/osh.asdl
}

gen-runtime-asdl() {
  local out=_devbuild/gen/runtime_asdl.py
  local import='from osh.meta import RUNTIME_TYPE_LOOKUP as TYPE_LOOKUP'
  PYTHONPATH=. asdl/gen_python.py core/runtime.asdl "$import" > $out
  echo "Wrote $out"
  gen-asdl-py-pickle core/runtime.asdl
}

# TODO: should fastlex.c be part of the dev build?  It means you need re2c
@@ -134,13 +143,19 @@ clean() {
# No fastlex, because we don't want to require re2c installation.
minimal() {
  mkdir -p _devbuild/gen
  # so osh_help.py and osh_asdl.py are importable

  rm -v _devbuild/gen/*

  # So modules are importable.
  touch _devbuild/__init__.py  _devbuild/gen/__init__.py

  gen-help
  gen-types-asdl
  gen-osh-asdl
  gen-runtime-asdl

  # BOOTSTRAP_LEVEL is a hack for avoiding circular dependencies.
  BOOTSTRAP_LEVEL=0 gen-types-asdl    # doesn't need Id
  BOOTSTRAP_LEVEL=1 gen-osh-asdl      # needs Id, which needs types.asdl
  BOOTSTRAP_LEVEL=2 gen-runtime-asdl  # ditto

  pylibc
}

osh/asdl_gen.py

@@ -0,0 +1,88 @@
#!/usr/bin/env python
from __future__ import print_function
"""
ast_gen.py: Generate the Id enum in C code.

# TODO: This should be renamed to asdl_gen.py
"""

import os
import pickle
import sys

from asdl import asdl_ as asdl
from asdl import front_end
from asdl import gen_cpp
from asdl import gen_python

def main(argv):
  try:
    action = argv[1]
  except IndexError:
    raise RuntimeError('Action required')

  try:
    schema_path = argv[2]
  except IndexError:
    raise RuntimeError('Schema path required')

  # To avoid circular dependencies, don't load Id for types.asdl.
  if os.path.basename(schema_path) == 'types.asdl':
    app_types = {}
  else:
    from osh.meta import Id
    app_types = {'id': asdl.UserType(Id)}

  if action == 'c':  # Generate C code for the lexer
    with open(schema_path) as f:
      schema_ast, _ = front_end.LoadSchema(f, app_types)

    v = gen_cpp.CEnumVisitor(sys.stdout)
    v.VisitModule(schema_ast)

  elif action == 'py':  # Generate Python code so we don't depend on ASDL schemas
    pickle_out_path = argv[3]

    with open(schema_path) as f:
      schema_ast, type_lookup = front_end.LoadSchema(f, app_types)

    f = sys.stdout

    f.write("""\
import pickle

from asdl import asdl_ as asdl
from asdl import const  # For const.NO_INTEGER
from asdl import py_meta

from core import util

f = util.GetResourceLoader().open('%s')
type_lookup = pickle.load(f)
TYPE_LOOKUP = asdl.TypeLookup(type_lookup)
f.close()

""" % pickle_out_path)

    v = gen_python.GenClassesVisitor(f)
    v.VisitModule(schema_ast)

    if pickle_out_path:
      # Pickle version 2 is better.  (Pickle version 0 uses
      # s.decode('string-escape')! )
      # In version 2, now I have 16 opcodes + STOP.
      with open(pickle_out_path, 'w') as f:
        pickle.dump(type_lookup.runtime_type_lookup, f, protocol=2)
      from core.util import log
      log('Wrote %s', pickle_out_path)

  else:
    raise RuntimeError('Invalid action %r' % action)


if __name__ == '__main__':
  try:
    main(sys.argv)
  except RuntimeError as e:
    print('FATAL: %s' % e, file=sys.stderr)
    sys.exit(1)