Separate and simplify the generation of ASDL reflection data.

Now there are separate _RuntimeType and AST hierarchies in
asdl/asdl_.py.  Prior to this change, they were oddly entangled.

- All unit tests pass
- All spec tests pass
- Tested 'test/wild.sh golden-subset'

asdl/arith_ast_test.py

@@ -133,7 +133,7 @@ def testProductType(self):
    s.length = 3
    print(s)

    assert isinstance(s.ASDL_TYPE, asdl_.Product)
    assert isinstance(s.ASDL_TYPE, asdl_.CompoundType)

    # Implementation detail for dynamic type checking
    assert isinstance(s, py_meta.CompoundObj)
@@ -150,7 +150,7 @@ def testSimpleSumType(self):
    assert isinstance(o, py_meta.SimpleObj)

    # Implementation detail for dynamic type checking
    assert isinstance(o.ASDL_TYPE, asdl_.Sum)
    assert isinstance(o.ASDL_TYPE, asdl_.SumType)

  def testCompoundSumType(self):
    print()
@@ -164,7 +164,7 @@ def testCompoundSumType(self):
    assert isinstance(c, py_meta.CompoundObj)

    # Implementation detail for dynamic type checking
    assert isinstance(c.ASDL_TYPE, asdl_.Constructor), c.ASDL_TYPE
    assert isinstance(c.ASDL_TYPE, asdl_.CompoundType), c.ASDL_TYPE

  def testOtherTypes(self):
    c = Const(66)

asdl/asdl_.py

@@ -36,6 +36,7 @@ def is_simple(sum):
      return False
  return True


#
# Type Descriptors
#
@@ -44,34 +45,43 @@ def is_simple(sum):
#
# Although we share Product and Sum.

class StrType(object):
class _RuntimeType(object):
  """A node hierarchy that exists at runtime."""
  pass


class StrType(_RuntimeType):
  def __repr__(self):
    return '<Str>'

class IntType(object):

class IntType(_RuntimeType):
  def __repr__(self):
    return '<Int>'

class BoolType(object):

class BoolType(_RuntimeType):
  def __repr__(self):
    return '<Bool>'


class ArrayType(object):
class ArrayType(_RuntimeType):
  def __init__(self, desc):
    self.desc = desc

  def __repr__(self):
    return '<Array %s>' % self.desc

class MaybeType(object):

class MaybeType(_RuntimeType):
  def __init__(self, desc):
    self.desc = desc  # another descriptor

  def __repr__(self):
    return '<Maybe %s>' % self.desc

class UserType(object):

class UserType(_RuntimeType):
  def __init__(self, typ):
    assert isinstance(typ, type), typ
    self.typ = typ
@@ -80,67 +90,61 @@ def __repr__(self):
    return '<UserType %s>' % self.typ


# TODO:
# Should we have our own ProductType, SumType, ConstructorType instead of
# attaching them below?
# It should be CompoundType.  ReflectNode.
#
# Two hierarchies:
# - AST
# - Reflection / ReflectNode / Mirror / _RuntimeType

class SumType(_RuntimeType):
  """Dummy node that doesn't require any reflection.

# TODO: Rename this to Reflection?
class TypeLookup(object):
  """Look up types by name.

  They are put in a flat namespace.
  obj.ASDL_TYPE points directly to the constructor, which you reflect on.
  """
  def __init__(self):
    pass

  def __init__(self, module, app_types=None):
    # Types that fields are declared with: int, id, word_part, etc.
    # Fields are NOT declared with Constructor names.
    self.declared_types = {}
  def __repr__(self):
    return '<SumType>'  # We need an entry for this but we don't use it?

    for d in module.dfns:
      self.declared_types[d.name] = d.value

    if app_types is not None:
      self.declared_types.update(app_types)
class CompoundType(_RuntimeType):
  """A product or Constructor instance.  Both have fields."""
  def __init__(self, fields):
    # List of (name, _RuntimeType) tuples.
    # NOTE: This list may be mutated after its set.
    self.fields = fields

    # Primitive types.
    self.declared_types.update(BUILTIN_TYPES)
  def __repr__(self):
    return '<CompoundType %s>' % self.fields

    # Types with fields that need to be reflected on: Product and Constructor.
    self.compound_types = {}
    for d in module.dfns:
      typ = d.value
      if isinstance(typ, Product):
        self.compound_types[d.name] = typ
      elif isinstance(typ, Sum):
        # e.g. 'assign_op' is simple, or 'bracket_op' is not simple.
        self.compound_types[d.name] = typ
  def GetFieldNames(self):
    for field_name, _ in self.fields:
      yield field_name

        for cons in typ.types:
          self.compound_types[cons.name] = cons
  def GetFields(self):
    for field_name, descriptor in self.fields:
      yield field_name, descriptor

  def ByFieldInstance(self, field):
  def LookupFieldType(self, field_name):
    """
    TODO: This is only used below?  And that part is only used by py_meta?
    py_meta is still useful though, because it has some dynamic type checking.
    I think I want to turn that back on.

    Args:
      field: Field() instance
    NOTE: Only used by py_meta.py.
    """
    t = self.declared_types[field.type]
    if field.seq:
      return ArrayType(t)
    for n, descriptor in self.fields:
      if n == field_name:
        return descriptor
    raise AssertionError(field_name)


BUILTIN_TYPES = {
    'string': StrType(),
    'int': IntType(),
    'bool': BoolType(),
}

    if field.opt:
      return MaybeType(t)

    return t
# TODO: Rename this to Reflection?
class TypeLookup(object):
  """Look up types by name.

  They are put in a flat namespace.
  """
  def __init__(self, runtime_type_lookup):
    self.runtime_type_lookup = runtime_type_lookup  # type name -> RuntimeType

  def ByTypeName(self, type_name):
    """Given a string, return a type descriptor.
@@ -149,27 +153,21 @@ def ByTypeName(self, type_name):
    Args:
      type_name: string, e.g. 'word_part' or 'LiteralPart'
    """
    if not type_name in self.compound_types:
      print('FATAL: %s' % self.compound_types.keys())
    return self.compound_types[type_name]
    #if not type_name in self.compound_types:
    #  print('FATAL: %s' % self.compound_types.keys())
    #return self.compound_types[type_name]
    if not type_name in self.runtime_type_lookup:
      print('FATAL: %s' % self.runtime_type_lookup.keys())
    return self.runtime_type_lookup[type_name]

  def __repr__(self):
    return repr(self.declared_types)
    return repr(self.runtime_type_lookup)


# The following classes define nodes into which the ASDL description is parsed.
# Note: this is a "meta-AST". ASDL files (such as Python.asdl) describe the AST
# structure used by a programming language. But ASDL files themselves need to be
# parsed. This module parses ASDL files and uses a simple AST to represent them.
# The following classes are the AST for the ASDL schema, i.e. the "meta-AST".
# See the EBNF at the top of the file to understand the logical connection
# between the various node types.

BUILTIN_TYPES = {
    'string': StrType(),
    'int': IntType(),
    'bool': BoolType(),
}

class AST(object):
    def Print(self, f, indent):
        raise NotImplementedError
@@ -232,7 +230,7 @@ def Print(self, f, indent):
        f.write('\n')


class _CompoundType(AST):
class _CompoundAST(AST):
    """Either a Product or Constructor.

    encode.py and format.py need a reflection API.
@@ -246,38 +244,10 @@ def __init__(self, fields):
        if self.fields:
          self.fields.append(Field('int', 'spids', seq=True))

        self.field_lookup = {f.name: f for f in self.fields}
        self.type_lookup = None  # set by ResolveTypes()

        self.type_cache = {}  # field name -> type descriptor

    def GetFieldNames(self):
        for f in self.fields:
          yield f.name

    def GetFields(self):
        for f in self.fields:
          field_name = f.name
          yield field_name, self.LookupFieldType(field_name)

    def LookupFieldType(self, field_name):
        """
        NOTE: Only used by py_meta.py.
        """
        # Cache and return it.  We don't want to create new instances every
        # time we iterate over the fields.
        try:
          return self.type_cache[field_name]
        except KeyError:
          field = self.field_lookup[field_name]
          desc = self.type_lookup.ByFieldInstance(field)
          self.type_cache[field_name] = desc
          return desc


class Constructor(_CompoundType):
class Constructor(_CompoundAST):
    def __init__(self, name, fields=None):
        _CompoundType.__init__(self, fields)
        _CompoundAST.__init__(self, fields)
        self.name = name

    def Print(self, f, indent):
@@ -308,9 +278,9 @@ def Print(self, f, indent):
        f.write('%s}\n' % ind)


class Product(_CompoundType):
class Product(_CompoundAST):
    def __init__(self, fields, attributes=None):
        _CompoundType.__init__(self, fields)
        _CompoundAST.__init__(self, fields)
        self.attributes = attributes or []

    def Print(self, f, indent):

asdl/asdl_demo.py

@@ -54,14 +54,8 @@ def main(argv):
        #marshal.dump(type_lookup, f)
        cPickle.dump(type_lookup, f)

      print('declared:')
      for name, desc in type_lookup.declared_types.items():
        print(name)
        print(desc)
      print()

      print('compound:')
      for name, desc in type_lookup.compound_types.items():
      print('runtime_type_lookup:')
      for name, desc in type_lookup.runtime_type_lookup.items():
        print(name)
        print(desc)
      print()