Preparing for a new ASDL reflection API.

This will make it easier to generate textual code while preserving
reflection for format.py and encode.py.

- Add a smoke test that does py_meta, format, encode for arith.asdl.
- Use the new asdl.ResolveTypes() API for osh/osh.asdl and
  core/runtime.asdl.

Other:

- Tweak pretty printing of ASDL AST
- meta.py: This may be deleted.  This is a draft of something more
  ambitious.

asdl/arith_ast.py

@@ -14,5 +14,6 @@

with open(schema_path) as f:
  module = asdl.parse(f)
type_lookup = asdl.ResolveTypes(module)
root = sys.modules[__name__]
py_meta.MakeTypes(module, root)
py_meta.MakeTypes(module, root, type_lookup)

asdl/asdl_.py

@@ -88,6 +88,59 @@ def __repr__(self):
    return '<UserType %s>' % self.typ


class TypeLookup(object):
  """Look up types by name.

  The names in a flat namespace.
  """

  def __init__(self, module, app_types=None):
    self.types = {}

    for typ in module.dfns:
      self.types[typ.name] = typ.value

    if app_types is not None:
      self.types.update(app_types)

    # TODO: Fold this in
    self.types.update(DESCRIPTORS_BY_NAME)

  def Get(self, type_name):
    return self.types[type_name]

  def __repr__(self):
    return repr(self.types)


def _CheckFieldsAndWire(typ, type_lookup):
  for f in typ.fields:
    # Will fail if it doesn't exist
    _ = type_lookup.Get(f.type)
  typ.type_lookup = type_lookup  # wire it for lookup


def ResolveTypes(module, app_types=None):
  # Walk the module, checking types, and adding the type_lookup attribute
  # everywhere.
  type_lookup = TypeLookup(module, app_types=app_types)

  for node in module.dfns:
    assert isinstance(node, Type), node
    v = node.value
    if isinstance(v, Product):
      _CheckFieldsAndWire(v, type_lookup)

    elif isinstance(v, Sum):
      for cons in v.types:
        _CheckFieldsAndWire(cons, type_lookup)

    else:
      raise AssertionError(typ)

  return type_lookup


# The following classes define nodes into which the ASDL description is parsed.
# Note: this is a "meta-AST". ASDL files (such as Python.asdl) describe the AST
# structure used by a programming language. But ASDL files themselves need to be
@@ -111,6 +164,8 @@ class Module(AST):
    def __init__(self, name, dfns):
        self.name = name
        self.dfns = dfns

        # NOTE: This is used for type checking.
        self.types = {type.name: type.value for type in dfns}

    def Print(self, f, indent):
@@ -122,6 +177,9 @@ def Print(self, f, indent):
          f.write('\n')
        f.write('%s}\n' % ind)

    def LookupFieldType(self, name):
        return self.types.get(name)


class Type(AST):
    def __init__(self, name, value):
@@ -135,18 +193,45 @@ def Print(self, f, indent):
        f.write('%s}\n' % ind)


class Constructor(AST):

class _CompoundType(AST):
    """Either a Product or Constructor.
    
    encode.py and format.py need a reflection API.
    """

    def __init__(self, fields):
        self.fields = fields or []
        self.field_lookup = {f.name: f.type for f in self.fields}
        self.type_lookup = None  # for runtime reflection

    def GetFields(self):
        for f in self.fields:
          field_name = f.name
          yield field_name, self.LookupFieldType(field_name)

    def LookupFieldType(self, field_name):
        type_name = self.field_lookup[field_name]
        return self.type_lookup.Get(type_name)



class Constructor(_CompoundType):
    def __init__(self, name, fields=None):
        _CompoundType.__init__(self, fields)
        self.name = name
        self.fields = fields or []

    def Print(self, f, indent):
        ind = indent * '  '
        f.write('%sConstructor %s {\n' % (ind, self.name))
        f.write('%sConstructor %s' % (ind, self.name))

        for field in self.fields:
          field.Print(f, indent+1)
        f.write('%s}\n' % ind)
        if self.fields:
          f.write(' {\n')
          for field in self.fields:
            field.Print(f, indent+1)
          f.write('%s}' % ind)

        f.write('\n')


class Field(AST):
@@ -168,7 +253,7 @@ def Print(self, f, indent):
        ind = indent * '  '
        f.write('%sField %s %s' % (ind, self.name, self.type))
        if extra:
          f.write('(')
          f.write(' (')
          f.write(', '.join(extra))
          f.write(')')
        f.write('\n')
@@ -189,9 +274,9 @@ def Print(self, f, indent):
        f.write('%s}\n' % ind)


class Product(AST):
class Product(_CompoundType):
    def __init__(self, fields, attributes=None):
        self.fields = fields
        _CompoundType.__init__(self, fields)
        self.attributes = attributes or []

    def Print(self, f, indent):
@@ -237,7 +322,7 @@ def __init__(self):
        super(Check, self).__init__()
        self.cons = {}
        self.errors = 0
        self.types = {}
        self.types = {}  # list of declared field types

    def visitModule(self, mod):
        for dfn in mod.dfns:

asdl/asdl_base.py

@@ -9,6 +9,11 @@
from core import util


#
# COMPATIBLE GENERATED code
#


# Copied from py_meta
class Obj(object):
  # NOTE: We're using CAPS for these static fields, since they are constant at
@@ -29,7 +34,7 @@ class CompoundObj(Obj):
  FIELDS = []  # ordered list of field names, overriden
  DESCRIPTOR_LOOKUP = {}  # field name: (asdl.Sum | asdl.Product | ...)

"""
  """
  def __repr__(self):
    # Breaking circular dependency, gah.
    from asdl import format as fmt
@@ -40,4 +45,33 @@ def __repr__(self):
    fmt.PrintTree(tree, ast_f)
    s, _ = ast_f.GetRaw()
    return s
    """
  """


#
# NEW metaprogramming API
#

# Copied from py_meta
class Obj2(object):
  ASDL_TYPE = None  # Used for type checking


# Do we need these?  I gues __repr__ is important.  But that can be generated
# then?  __init__ and __repr__ need to be generated.
# Or I guess __repr__ can always be fmt.PrintTree(...)?
# You are using this for enums too, so __repr__ has to be hooked up.

#class SimpleObj2(Obj2):
#  def __init__(self, enum_id, sum_type_name):
#    self.enum_id = enum_id
#    self.name = sum_type_name
#
#  def __repr__(self):
#    # NOTE: This be self.__bases__[0].__name__?  There's probably no advantage
#    # to that though?
#    return '<%s %s %s>' % (self.__class__.__name__, self.name, self.enum_id)


#class CompoundObj2(Obj2):
#  pass

asdl/asdl_demo.py

@@ -12,6 +12,7 @@
from asdl import format as fmt

from core.id_kind import Id
from core.util import log


def main(argv):
@@ -26,21 +27,25 @@ def main(argv):
    with open(schema_path) as f:
      module = asdl.parse(f)

    app_types = {'id': asdl.UserType(Id)}
    type_lookup = asdl.ResolveTypes(module, app_types)

    # Note this is a big tree.  But we really want a graph of pointers to
    # instances.
    # Type(name, Product(...))
    # Type(name, Sum([Constructor(...), ...]))
    print(module)
    #print(module)

    root = sys.modules[__name__]
    # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
    # demo.
    py_meta.MakeTypes(module, root, app_types={'id': asdl.UserType(Id)})
    py_meta.MakeTypes(module, root, type_lookup)

    print('Dynamically created a Python module with these types:')
    for name in dir(root):
      print('\t' + name)

  elif action == 'arith-encode':
  elif action == 'arith-encode':  # oheap encoding
    expr = argv[2]
    out_path = argv[3]

@@ -53,7 +58,7 @@ def main(argv):
      out = encode.BinOutput(f)
      encode.EncodeRoot(obj, enc, out)

  elif action == 'arith-format':
  elif action == 'arith-format':  # pretty printing
    expr = argv[2]

    obj = arith_parse.ParseShell(expr)
@@ -62,18 +67,11 @@ def main(argv):
    #treee= ['hi', 'there', ['a', 'b'], 'c']
    f = fmt.DetectConsoleOutput(sys.stdout)
    fmt.PrintTree(tree, f)
    print()

    # Might need to print the output?
    # out.WriteToFile?

  elif action == 'repr':
    # Hm this isn't valid Python code, but we can change it to be as a hack.
    field1 = asdl.Field('str', 'name')
    field2 = asdl.Field('int', 'age')
    print(repr(field1))
    s = asdl.Sum([field1, field2])
    print(repr(s))

  else:
    raise RuntimeError('Invalid action %r' % action)

@@ -82,5 +80,5 @@ def main(argv):
  try:
    main(sys.argv)
  except RuntimeError as e:
    print('FATAL: %s' % e, file=sys.stderr)
    print('FATAL: %r' % e, file=sys.stderr)
    sys.exit(1)

asdl/format_test.py

@@ -16,6 +16,22 @@

class FormatTest(unittest.TestCase):

  def testSimpleSum(self):
    node = arith_ast.op_id_e.Plus
    print(node)

    f = cStringIO.StringIO()
    ast_f = fmt.TextOutput(f)

    tree = fmt.MakeTree(node)
    fmt.PrintTree(tree, ast_f)

    # Hm this prints 'Plus'.  Doesn't print the class or the number.
    # But those aren't intrinsic.  These are mostly used for ther IDENTITY.
    # I think the ASDL_TYPE field contains the relevant info.  Yes!
    pretty_str = f.getvalue()
    print(pretty_str)

  def testRepeatedString(self):
    node = arith_ast.assign('declare', ['-r', '-x'])