Minor refactoring / renaming of ASDL.

A bunch of warts have accumulated here.
oilshell · Aug 18, 2018 · b8086729813aa7798426fa12babddb7fe83a72d5 · b808672
1 parent 53de755
commit b8086729813aa7798426fa12babddb7fe83a72d5
Unified Split

Showing with 85 additions and 60 deletions.

+45 −28 asdl/asdl_.py

+6 −2 asdl/asdl_demo.py

+25 −24 asdl/front_end.py

+6 −6 asdl/gen_cpp.py

+3 −0 osh/meta.py
diff --git a/asdl/asdl_.py b/asdl/asdl_.py
@@ -36,6 +36,13 @@ def is_simple(sum):
      return False
  return True

+#
+# Type Descriptors
+#
+# These are more convenient than using the AST directly, since it still has
+# string type names?
+#
+# Although we share Product and Sum.

 class StrType(object):
  def __repr__(self):
@@ -72,6 +79,10 @@ def __init__(self, typ):
  def __repr__(self):
    return '<UserType %s>' % self.typ

+# TODO:
+# Should we have our own ProductType, SumType, ConstructorType instead of
+# attaching them below?
+

 # The following classes define nodes into which the ASDL description is parsed.
 # Note: this is a "meta-AST". ASDL files (such as Python.asdl) describe the AST
@@ -80,7 +91,11 @@ def __repr__(self):
 # See the EBNF at the top of the file to understand the logical connection
 # between the various node types.

-builtin_types = {'string', 'int', 'bool'}
+BUILTIN_TYPES = {
+    'string': StrType(),
+    'int': IntType(),
+    'bool': BoolType(),
+}

 class AST(object):
    def Print(self, f, indent):
@@ -119,6 +134,31 @@ def Print(self, f, indent):
        f.write('%s}\n' % ind)


+class Field(AST):
+    def __init__(self, type, name=None, seq=False, opt=False):
+        self.type = type
+        self.name = name
+        self.seq = seq
+        self.opt = opt
+
+    def Print(self, f, indent):
+        extra = []
+        if self.seq:
+            extra.append('seq=True')
+        elif self.opt:
+            extra.append('opt=True')
+        else:
+            extra = ""
+
+        ind = indent * '  '
+        f.write('%sField %s %s' % (ind, self.name, self.type))
+        if extra:
+          f.write(' (')
+          f.write(', '.join(extra))
+          f.write(')')
+        f.write('\n')
+
+
 class _CompoundType(AST):
    """Either a Product or Constructor.

@@ -136,7 +176,7 @@ def __init__(self, fields):
        self.field_lookup = {f.name: f for f in self.fields}
        self.type_lookup = None  # set by ResolveTypes()

-        self.type_cache = {}
+        self.type_cache = {}  # field name -> type descriptor

    def GetFieldNames(self):
        for f in self.fields:
@@ -148,6 +188,9 @@ def GetFields(self):
          yield field_name, self.LookupFieldType(field_name)

    def LookupFieldType(self, field_name):
+        """
+        NOTE: Only used by py_meta.py.
+        """
        # Cache and return it.  We don't want to create new instances every
        # time we iterate over the fields.
        try:
@@ -177,31 +220,6 @@ def Print(self, f, indent):
        f.write('\n')


-class Field(AST):
-    def __init__(self, type, name=None, seq=False, opt=False):
-        self.type = type
-        self.name = name
-        self.seq = seq
-        self.opt = opt
-
-    def Print(self, f, indent):
-        extra = []
-        if self.seq:
-            extra.append('seq=True')
-        elif self.opt:
-            extra.append('opt=True')
-        else:
-            extra = ""
-
-        ind = indent * '  '
-        f.write('%sField %s %s' % (ind, self.name, self.type))
-        if extra:
-          f.write(' (')
-          f.write(', '.join(extra))
-          f.write(')')
-        f.write('\n')
-
-
 class Sum(AST):
    def __init__(self, types, attributes=None):
        self.types = types  # List[Constructor]
@@ -230,4 +248,3 @@ def Print(self, f, indent):
        if self.attributes:
          f.write('%s\n' % self.attributes)
        f.write('%s}\n' % ind)
-
diff --git a/asdl/asdl_demo.py b/asdl/asdl_demo.py
@@ -30,12 +30,16 @@ def main(argv):
    schema_path = argv[2]
    app_types = {'id': asdl.UserType(Id)}
    with open(schema_path) as f:
-      module, type_lookup = front_end.LoadSchema(f, app_types)
+      schema_ast, type_lookup = front_end.LoadSchema(f, app_types)

    root = sys.modules[__name__]
    # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
    # demo.
-    py_meta.MakeTypes(module, root, type_lookup)
+    py_meta.MakeTypes(schema_ast, root, type_lookup)
+
+    log('AST for this ASDL schema:')
+    schema_ast.Print(sys.stdout, 0)
+    print()

    log('Dynamically created a Python module with these types:')
    for name in dir(root):
diff --git a/asdl/front_end.py b/asdl/front_end.py
@@ -10,7 +10,7 @@


 # Types for describing tokens in an ASDL specification.
-class TokenKind:
+class TokenKind(object):
    """TokenKind is provides a scope for enumerated token kinds."""
    (ConstructorId, TypeId, Equals, Comma, Question, Pipe, Asterisk,
     LParen, RParen, LBrace, RBrace) = range(11)
@@ -19,7 +19,7 @@ class TokenKind:
        '=': Equals, ',': Comma,    '?': Question, '|': Pipe,    '(': LParen,
        ')': RParen, '*': Asterisk, '{': LBrace,   '}': RBrace}

-class Token:
+class Token(object):
    def __init__(self, kind, value, lineno):
        self.kind = kind
        self.value = value
@@ -33,7 +33,7 @@ def __init__(self, msg, lineno=None):
    def __str__(self):
        return 'Syntax error on line {0.lineno}: {0.msg}'.format(self)

-def tokenize_asdl(f):
+def _Tokenize(f):
    """Tokenize the given buffer. Yield Token objects."""
    for lineno, line in enumerate(f, 1):
        for m in re.finditer(r'\s*(\w+|--.*|.)', line.strip()):
@@ -55,11 +55,11 @@ def tokenize_asdl(f):
                    raise ASDLSyntaxError('Invalid operator %s' % c, lineno)
                yield Token(op_kind, c, lineno)

-class ASDLParser:
+class ASDLParser(object):
    """Parser for ASDL files.

    Create, then call the parse method on a buffer containing ASDL.
-    This is a simple recursive descent parser that uses tokenize_asdl for the
+    This is a simple recursive descent parser that uses _Tokenize for the
    lexing.
    """
    def __init__(self):
@@ -69,7 +69,7 @@ def __init__(self):
    def parse(self, f):
        """Parse the ASDL in the file and return an AST with a Module root.
        """
-        self._tokenizer = tokenize_asdl(f)
+        self._tokenizer = _Tokenize(f)
        self._advance()
        return self._parse_module()

@@ -266,18 +266,6 @@ def visitProduct(self, prod, name):
            self.visit(f, name)


-def _CheckSchema(mod, app_types=None):
-    """Check the parsed ASDL tree for correctness.
-
-    Return True if success. For failure, the errors are printed out and False
-    is returned.
-    """
-    app_types = app_types or {}
-    v = Check()
-    v.visit(mod)
-    return not v.errors
-
-
 class TypeLookup(object):
  """Look up types by name.

@@ -296,9 +284,7 @@ def __init__(self, module, app_types=None):
      self.declared_types.update(app_types)

    # Primitive types.
-    self.declared_types['string'] = asdl.StrType()
-    self.declared_types['int'] = asdl.IntType()
-    self.declared_types['bool'] = asdl.BoolType()
+    self.declared_types.update(asdl.BUILTIN_TYPES)

    # Types with fields that need to be reflected on: Product and Constructor.
    self.compound_types = {}
@@ -315,6 +301,10 @@ def __init__(self, module, app_types=None):

  def ByFieldInstance(self, field):
    """
+    TODO: This is only used below?  And that part is only used by py_meta?
+    py_meta is still useful though, because it has some dynamic type checking.
+    I think I want to turn that back on.
+
    Args:
      field: Field() instance
    """
@@ -328,7 +318,9 @@ def ByFieldInstance(self, field):
    return t

  def ByTypeName(self, type_name):
-    """
+    """Given a string, return a type descriptor.
+
+    Used by generated code, e.g. in _devbuild/gen/osh_asdl.py.
    Args:
      type_name: string, e.g. 'word_part' or 'LiteralPart'
    """
@@ -341,9 +333,15 @@ def __repr__(self):


 def _CheckFieldsAndWire(typ, type_lookup):
+  """
+  Given a compound type, iterate
+  """
  for f in typ.fields:
    # Will fail if it doesn't exist
    _ = type_lookup.ByFieldInstance(f)
+
+  # _CompoundType instances have this attribute
+  assert hasattr(typ, 'type_lookup'), typ
  typ.type_lookup = type_lookup  # wire it for lookup


@@ -383,8 +381,11 @@ def LoadSchema(f, app_types):
  p = ASDLParser()
  schema_ast = p.parse(f)

-  if not _CheckSchema(schema_ast, app_types):
-    raise AssertionError('ASDL file is invalid')
+  v = Check()
+  v.visit(schema_ast)
+
+  if v.errors:
+    raise AssertionError('ASDL file is invalid: %s' % v.errors)

  type_lookup = _MakeReflectionObject(schema_ast, app_types)
  return schema_ast, type_lookup
diff --git a/asdl/gen_cpp.py b/asdl/gen_cpp.py
@@ -139,9 +139,9 @@ def Emit(s, depth=depth):

    # rudimentary attribute handling
    for field in sum.attributes:
-      type = str(field.type)
-      assert type in asdl.builtin_types, type
-      Emit("%s %s;" % (type, field.name), depth + 1)
+      type_name = str(field.type)
+      assert type_name in asdl.BUILTIN_TYPES, type_name
+      Emit("%s %s;" % (type_name, field.name), depth + 1)

  def VisitConstructor(self, cons, def_name, depth):
    #print(dir(cons))
@@ -166,9 +166,9 @@ def VisitProduct(self, product, name, depth):

    for field in product.attributes:
      # rudimentary attribute handling
-      type = str(field.type)
-      assert type in asdl.builtin_types, type
-      self.Emit("%s %s;" % (type, field.name), depth + 1)
+      type_name = str(field.type)
+      assert type_name in asdl.BUILTIN_TYPES, type_name
+      self.Emit("%s %s;" % (type_name, field.name), depth + 1)
    self.Emit("};", depth)
    self.Emit("", depth)

diff --git a/osh/meta.py b/osh/meta.py
@@ -12,7 +12,10 @@
 """

 from asdl import asdl_ as asdl
+
+# TODO: Remove this dependency!
 from asdl import front_end
+
 from asdl import py_meta

 from core import id_kind