ASDL reflection now works with generated ASDL classes.

- New TypeLookup API for generated code.
- Cache field descriptors by field name inside _CompoundType, so we
  don't create new instances every time we iterate.
- Fix gen_cpp to use the new TypeLookup style.

The generated code speeds up parsing significantly.  I will measure this
shortly.  It's not switched on yet because we have to take care of the
dev build.

asdl/asdl_.py

@@ -54,17 +54,6 @@ def __repr__(self):
    return '<Bool>'


# NOTE: We might want to change this to PrimitiveType(int tag).  Then
# ArrayType, MaybeType, Sum, and Product are expressions involving primitives.

# The type that should be passed.
DESCRIPTORS_BY_NAME = {
    'string': StrType(),
    'int': IntType(),
    'bool': BoolType(),
}


class ArrayType:
  def __init__(self, desc):
    self.desc = desc
@@ -95,19 +84,40 @@ class TypeLookup(object):
  """

  def __init__(self, module, app_types=None):
    self.types = {}
    # types that fields are declared with: int, id, word_part, etc.
    # Fields are not declared with constructor names.
    self.declared_types = {}

    for typ in module.dfns:
      self.types[typ.name] = typ.value
    for d in module.dfns:
      self.declared_types[d.name] = d.value

    if app_types is not None:
      self.types.update(app_types)

    # TODO: Don't need a public constant.
    self.types.update(DESCRIPTORS_BY_NAME)

  def Get(self, field):
    t = self.types[field.type]
      self.declared_types.update(app_types)

    # Primitive types.
    self.declared_types['string'] = StrType()
    self.declared_types['int'] =  IntType()
    self.declared_types['bool'] = BoolType()

    # Types with fields that need to be reflected on: Product and Constructor.
    self.compound_types = {}
    for d in module.dfns:
      typ = d.value
      if isinstance(typ, Product):
        self.compound_types[d.name] = typ
      elif isinstance(typ, Sum):
        # e.g. 'assign_op' is simple, or 'bracket_op' is not simple.
        self.compound_types[d.name] = typ

        for cons in typ.types:
          self.compound_types[cons.name] = cons

  def ByFieldInstance(self, field):
    """
    Args:
      field: Field() instance
    """
    t = self.declared_types[field.type]
    if field.seq:
      return ArrayType(t)

@@ -116,14 +126,23 @@ def Get(self, field):

    return t

  def ByTypeName(self, type_name):
    """
    Args:
      type_name: string, e.g. 'word_part' or 'LiteralPart'
    """
    if not type_name in self.compound_types:
      print 'FATAL', self.compound_types.keys()
    return self.compound_types[type_name]

  def __repr__(self):
    return repr(self.types)
    return repr(self.declared_types)


def _CheckFieldsAndWire(typ, type_lookup):
  for f in typ.fields:
    # Will fail if it doesn't exist
    _ = type_lookup.Get(f)
    _ = type_lookup.ByFieldInstance(f)
  typ.type_lookup = type_lookup  # wire it for lookup


@@ -173,9 +192,6 @@ def __init__(self, name, dfns):
        self.name = name
        self.dfns = dfns

        # NOTE: This is used for type checking.
        self.types = {type.name: type.value for type in dfns}

    def Print(self, f, indent):
        ind = indent * '  '
        f.write('%sModule %s {\n' % (ind, self.name))
@@ -185,9 +201,6 @@ def Print(self, f, indent):
          f.write('\n')
        f.write('%s}\n' % ind)

    def LookupFieldType(self, name):
        return self.types.get(name)


class Type(AST):
    def __init__(self, name, value):
@@ -217,22 +230,29 @@ def __init__(self, fields):
          self.fields.append(Field('int', 'spids', seq=True))

        self.field_lookup = {f.name: f for f in self.fields}
        self.type_lookup = None  # for runtime reflection
        self.type_lookup = None  # set by ResolveTypes()

    def GetFields(self):
        for f in self.fields:
          field_name = f.name
          yield field_name, self.LookupFieldType(field_name)
        self.type_cache = {}

    def GetFieldNames(self):
        """Only used by core/test_lib.py."""
        for f in self.fields:
          yield f.name

    def LookupFieldType(self, field_name):
        field = self.field_lookup[field_name]
        return self.type_lookup.Get(field)
    def GetFields(self):
        for f in self.fields:
          field_name = f.name
          yield field_name, self.LookupFieldType(field_name)

    def LookupFieldType(self, field_name):
        # Cache and return it.  We don't want to create new instances every
        # time we iterate over the fields.
        try:
          return self.type_cache[field_name]
        except KeyError:
          field = self.field_lookup[field_name]
          desc = self.type_lookup.ByFieldInstance(field)
          self.type_cache[field_name] = desc
          return desc


class Constructor(_CompoundType):
@@ -387,13 +407,6 @@ def check(mod, app_types=None):
    app_types = app_types or {}
    v = Check()
    v.visit(mod)

    for t in v.types:
        if t in mod.types or t in builtin_types or t in app_types:
            continue
        v.errors += 1
        uses = ", ".join(v.types[t])
        print('Undefined type {}, used in {}'.format(t, uses))
    return not v.errors

# The ASDL parser itself comes next. The only interesting external interface

asdl/gen_cpp.py

@@ -112,36 +112,12 @@ def VisitModule(self, module):
class AsdlVisitor:
  def __init__(self, f):
    self.f = f
    self.module = None

  def GetCppType(self, field):
    """Return a string for the C++ name of the type."""
    type_name = field.type

    cpp_type = _BUILTINS.get(type_name)
    if cpp_type is not None:
      return cpp_type

    typ = self.module.types[type_name]
    if isinstance(typ, asdl.Sum) and asdl.is_simple(typ):
      # Use the enum instead of the class.
      return "%s_e" % type_name

    # - Pointer for optional type.
    # - ints and strings should generally not be optional?  We don't have them
    # in osh yet, so leave it out for now.
    if field.opt:
      return "%s_t*" % type_name

    return "%s_t&" % type_name

  def Emit(self, s, depth, reflow=True):
    for line in FormatLines(s, depth):
      self.f.write(line)

  def VisitModule(self, mod):
    self.module = mod  # Save it for GetCppType to look up types.

    for dfn in mod.dfns:
      self.VisitType(dfn)
    self.EmitFooter()
@@ -189,13 +165,35 @@ def EmitFooter(self):
class ClassDefVisitor(AsdlVisitor):
  """Generate C++ classes and type-safe enums."""

  def __init__(self, f, enc_params, enum_types=None):
  def __init__(self, f, enc_params, type_lookup, enum_types=None):
    AsdlVisitor.__init__(self, f)
    self.ref_width = enc_params.ref_width
    self.type_lookup = type_lookup
    self.enum_types = enum_types or {}
    self.pointer_type = enc_params.pointer_type
    self.footer = []  # lines

  def _GetCppType(self, field):
    """Return a string for the C++ name of the type."""
    type_name = field.type

    cpp_type = _BUILTINS.get(type_name)
    if cpp_type is not None:
      return cpp_type

    typ = self.type_lookup.ByTypeName(type_name)
    if isinstance(typ, asdl.Sum) and asdl.is_simple(typ):
      # Use the enum instead of the class.
      return "%s_e" % type_name

    # - Pointer for optional type.
    # - ints and strings should generally not be optional?  We don't have them
    # in osh yet, so leave it out for now.
    if field.opt:
      return "%s_t*" % type_name

    return "%s_t&" % type_name

  def EmitFooter(self):
    for line in self.footer:
      self.f.write(line)
@@ -278,7 +276,7 @@ def VisitField(self, field, type_name, offset, depth):

    http://stackoverflow.com/questions/5808758/why-is-a-static-cast-from-a-pointer-to-base-to-a-pointer-to-derived-invalid
    """
    ctype = self.GetCppType(field)
    ctype = self._GetCppType(field)
    name = field.name
    pointer_type = self.pointer_type
    # Either 'left' or 'BoolBinary::left', depending on whether it's inline.
@@ -408,6 +406,7 @@ def main(argv):
    schema_path = argv[2]
    with open(schema_path) as input_f:
      module = asdl.parse(input_f)
    type_lookup = asdl.ResolveTypes(module)

    f = sys.stdout

@@ -448,7 +447,7 @@ class Obj {
    # Id should be treated as an enum.
    c = ChainOfVisitors(
        ForwardDeclareVisitor(f),
        ClassDefVisitor(f, enc, enum_types=['Id']))
        ClassDefVisitor(f, enc, type_lookup, enum_types=['Id']))
    c.VisitModule(module)

    f.write("""\

asdl/gen_python.py

@@ -4,7 +4,8 @@

Generate Python code from and ASDL schema.

TODO: What about Id?  app_types?
TODO:
- What about Id?  app_types?
"""

import sys
@@ -15,7 +16,6 @@

class GenClassesVisitor(gen_cpp.AsdlVisitor):
  # TODO:
  # - __eq__ isn't the same
  # - DESCRIPTOR  and FIELDS are dummies right now.
  # - I think FIELDS is used for encoding.
  #
@@ -25,8 +25,8 @@ class GenClassesVisitor(gen_cpp.AsdlVisitor):
  #     but also in _Init?

  def VisitSimpleSum(self, sum, name, depth):
    self.Emit('class %s_e(asdl_base.SimpleObj):' % name, depth)
    self.Emit('  pass', depth)
    self.Emit('class %s_e(py_meta.SimpleObj):' % name, depth)
    self.Emit('  ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % name, depth)
    self.Emit('', depth)

    # Just use #define, since enums aren't namespaced.
@@ -52,15 +52,13 @@ def _GenClass(self, desc, name, super_name, depth, tag_num=None,
    # oheap serialization.  TODO: measure the effect of __slots__, and then get
    # rid of FIELDS?  Or you can just make it an alias.
    # FIELDS = self.__slots__.
    self.Emit('  FIELDS = %s' % quoted_fields, depth)

    self.Emit('  ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % name, depth)
    self.Emit('  __slots__ = %s' % quoted_fields, depth)

    # TODO: 
    # py_meta.MakeTypes and py_meta._MakeFieldDescriptors fill
    # DESCRIPTOR_LOOKUP, which is used for pretty printing.
    lookup = {}
    self.Emit('  DESCRIPTOR_LOOKUP = %r' % lookup, depth)
    self.Emit('', depth)

    args = ', '.join('%s=None' % f.name for f in desc.fields)
@@ -99,6 +97,7 @@ def VisitConstructor(self, cons, def_name, tag_num, depth):
      self._GenClass(cons, cons.name, def_name, depth, tag_num=tag_num)
    else:
      self.Emit("class %s(%s):" % (cons.name, def_name), depth)
      self.Emit('  ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % cons.name, depth)
      self.Emit('  tag = %d'  % tag_num, depth)
      self.Emit('', depth)

@@ -109,8 +108,8 @@ def VisitCompoundSum(self, sum, name, depth):
      self.Emit('  %s = %d' % (variant.name, i + 1), depth)
    self.Emit('', depth)

    self.Emit('class %s(asdl_base.CompoundObj):' % name, depth)
    self.Emit('  pass', depth)
    self.Emit('class %s(py_meta.CompoundObj):' % name, depth)
    self.Emit('  ASDL_TYPE = TYPE_LOOKUP.ByTypeName(%r)' % name, depth)
    self.Emit('', depth)

    # define command_t, and then make subclasses
@@ -120,7 +119,7 @@ def VisitCompoundSum(self, sum, name, depth):
      self.VisitConstructor(t, super_name, tag_num, depth)

  def VisitProduct(self, product, name, depth):
    self._GenClass(product, name, 'asdl_base.CompoundObj', depth)
    self._GenClass(product, name, 'py_meta.CompoundObj', depth)

  def EmitFooter(self):
    pass
@@ -136,7 +135,9 @@ def main(argv):

  f.write("""\
from asdl import const  # For const.NO_INTEGER
from asdl import asdl_base
from asdl import py_meta
from osh.ast_ import TYPE_LOOKUP

""")

  v = GenClassesVisitor(f)

asdl/py_meta.py

@@ -121,9 +121,17 @@ def __repr__(self):


class CompoundObj(Obj):
  """A compound object with fields, e.g. a Product or Constructor.
  # TODO: Remove this?

  Uses some metaprogramming.
  # Always set for constructor types, which are subclasses of sum types.  Never
  # set for product types.
  tag = None


class DebugCompoundObj(CompoundObj):
  """A CompoundObj that does dynamic type checks.

  Used by MakeTypes().
  """
  # Always set for constructor types, which are subclasses of sum types.  Never
  # set for product types.
@@ -253,7 +261,7 @@ def MakeTypes(module, root, type_lookup):

        # e.g. for arith_expr
        # Should this be arith_expr_t?  It is in C++.
        base_class = type(defn.name, (CompoundObj, ), {})
        base_class = type(defn.name, (DebugCompoundObj, ), {})
        setattr(root, defn.name, base_class)

        # Make a type and a enum tag for each alternative.
@@ -277,7 +285,7 @@ def MakeTypes(module, root, type_lookup):

    elif isinstance(typ, asdl.Product):
      class_attr = {'ASDL_TYPE': typ}
      cls = type(defn.name, (CompoundObj, ), class_attr)
      cls = type(defn.name, (DebugCompoundObj, ), class_attr)
      setattr(root, defn.name, cls)

    else: