Preparing to break the runtime dependency on asdl/front_end.py.

- Move TypeLookup class to the asdl_.py module, which will be available at runtime. - Shell functions to test that pretty-printing doesn't regress using wild data. - Tweaks to oheap shell functions
oilshell · Aug 18, 2018 · 56f4fac90a6a70b3fab997de9993dc69bed36d96 · 56f4fac
1 parent b808672
commit 56f4fac90a6a70b3fab997de9993dc69bed36d96
Unified Split

Showing with 127 additions and 85 deletions.

+73 −0 asdl/asdl_.py

+21 −0 asdl/asdl_demo.py

+19 −83 asdl/front_end.py

+3 −2 benchmarks/oheap.sh

+11 −0 test/wild.sh
diff --git a/asdl/asdl_.py b/asdl/asdl_.py
@@ -79,9 +79,82 @@ def __init__(self, typ):
  def __repr__(self):
    return '<UserType %s>' % self.typ

+
 # TODO:
 # Should we have our own ProductType, SumType, ConstructorType instead of
 # attaching them below?
+# It should be CompoundType.  ReflectNode.
+#
+# Two hierarchies:
+# - AST
+# - Reflection / ReflectNode / Mirror / _RuntimeType
+
+
+# TODO: Rename this to Reflection?
+class TypeLookup(object):
+  """Look up types by name.
+
+  They are put in a flat namespace.
+  """
+
+  def __init__(self, module, app_types=None):
+    # Types that fields are declared with: int, id, word_part, etc.
+    # Fields are NOT declared with Constructor names.
+    self.declared_types = {}
+
+    for d in module.dfns:
+      self.declared_types[d.name] = d.value
+
+    if app_types is not None:
+      self.declared_types.update(app_types)
+
+    # Primitive types.
+    self.declared_types.update(BUILTIN_TYPES)
+
+    # Types with fields that need to be reflected on: Product and Constructor.
+    self.compound_types = {}
+    for d in module.dfns:
+      typ = d.value
+      if isinstance(typ, Product):
+        self.compound_types[d.name] = typ
+      elif isinstance(typ, Sum):
+        # e.g. 'assign_op' is simple, or 'bracket_op' is not simple.
+        self.compound_types[d.name] = typ
+
+        for cons in typ.types:
+          self.compound_types[cons.name] = cons
+
+  def ByFieldInstance(self, field):
+    """
+    TODO: This is only used below?  And that part is only used by py_meta?
+    py_meta is still useful though, because it has some dynamic type checking.
+    I think I want to turn that back on.
+
+    Args:
+      field: Field() instance
+    """
+    t = self.declared_types[field.type]
+    if field.seq:
+      return ArrayType(t)
+
+    if field.opt:
+      return MaybeType(t)
+
+    return t
+
+  def ByTypeName(self, type_name):
+    """Given a string, return a type descriptor.
+
+    Used by generated code, e.g. in _devbuild/gen/osh_asdl.py.
+    Args:
+      type_name: string, e.g. 'word_part' or 'LiteralPart'
+    """
+    if not type_name in self.compound_types:
+      print('FATAL: %s' % self.compound_types.keys())
+    return self.compound_types[type_name]
+
+  def __repr__(self):
+    return repr(self.declared_types)


 # The following classes define nodes into which the ASDL description is parsed.
diff --git a/asdl/asdl_demo.py b/asdl/asdl_demo.py
@@ -45,6 +45,27 @@ def main(argv):
    for name in dir(root):
      print('\t' + name)

+    if 1:
+      # NOTE: It can be pickled, but not marshaled
+      import marshal
+      import cPickle
+      print(dir(marshal))
+      with open('out.marshal', 'w') as f:
+        #marshal.dump(type_lookup, f)
+        cPickle.dump(type_lookup, f)
+
+      print('declared:')
+      for name, desc in type_lookup.declared_types.items():
+        print(name)
+        print(desc)
+      print()
+
+      print('compound:')
+      for name, desc in type_lookup.compound_types.items():
+        print(name)
+        print(desc)
+      print()
+
  elif action == 'arith-encode':  # oheap encoding
    expr = argv[2]
    out_path = argv[3]
diff --git a/asdl/front_end.py b/asdl/front_end.py
@@ -266,76 +266,8 @@ def visitProduct(self, prod, name):
            self.visit(f, name)


-class TypeLookup(object):
-  """Look up types by name.
-
-  The names in a flat namespace.
-  """
-
-  def __init__(self, module, app_types=None):
-    # types that fields are declared with: int, id, word_part, etc.
-    # Fields are not declared with constructor names.
-    self.declared_types = {}
-
-    for d in module.dfns:
-      self.declared_types[d.name] = d.value
-
-    if app_types is not None:
-      self.declared_types.update(app_types)
-
-    # Primitive types.
-    self.declared_types.update(asdl.BUILTIN_TYPES)
-
-    # Types with fields that need to be reflected on: Product and Constructor.
-    self.compound_types = {}
-    for d in module.dfns:
-      typ = d.value
-      if isinstance(typ, Product):
-        self.compound_types[d.name] = typ
-      elif isinstance(typ, Sum):
-        # e.g. 'assign_op' is simple, or 'bracket_op' is not simple.
-        self.compound_types[d.name] = typ
-
-        for cons in typ.types:
-          self.compound_types[cons.name] = cons
-
-  def ByFieldInstance(self, field):
-    """
-    TODO: This is only used below?  And that part is only used by py_meta?
-    py_meta is still useful though, because it has some dynamic type checking.
-    I think I want to turn that back on.
-
-    Args:
-      field: Field() instance
-    """
-    t = self.declared_types[field.type]
-    if field.seq:
-      return asdl.ArrayType(t)
-
-    if field.opt:
-      return asdl.MaybeType(t)
-
-    return t
-
-  def ByTypeName(self, type_name):
-    """Given a string, return a type descriptor.
-
-    Used by generated code, e.g. in _devbuild/gen/osh_asdl.py.
-    Args:
-      type_name: string, e.g. 'word_part' or 'LiteralPart'
-    """
-    if not type_name in self.compound_types:
-      print('FATAL: %s' % self.compound_types.keys())
-    return self.compound_types[type_name]
-
-  def __repr__(self):
-    return repr(self.declared_types)
-
-
 def _CheckFieldsAndWire(typ, type_lookup):
-  """
-  Given a compound type, iterate
-  """
+  """Create back pointers on a _CompoundType."""
  for f in typ.fields:
    # Will fail if it doesn't exist
    _ = type_lookup.ByFieldInstance(f)
@@ -353,20 +285,24 @@ def _MakeReflectionObject(module, app_types=None):
  Args:
    module: Module node, root of the AST for a schema.
  """
-  type_lookup = TypeLookup(module, app_types=app_types)
-
-  for node in module.dfns:
-    assert isinstance(node, Type), node
-    v = node.value
-    if isinstance(v, Product):
-      _CheckFieldsAndWire(v, type_lookup)
-
-    elif isinstance(v, Sum):
-      for cons in v.types:
-        _CheckFieldsAndWire(cons, type_lookup)
-
-    else:
-      raise AssertionError(v)
+  type_lookup = asdl.TypeLookup(module, app_types=app_types)
+
+  # MAKE BACK POINTERS.  From the _CompoundType instance to the type_lookup
+  # instance.
+  # TODO: Fix this because it makes it hard to serialize reflection metadata!
+  if 1:
+    for node in module.dfns:
+      assert isinstance(node, Type), node
+      v = node.value
+      if isinstance(v, Product):
+        _CheckFieldsAndWire(v, type_lookup)
+
+      elif isinstance(v, Sum):
+        for cons in v.types:
+          _CheckFieldsAndWire(cons, type_lookup)
+
+      else:
+        raise AssertionError(v)

  return type_lookup

diff --git a/benchmarks/oheap.sh b/benchmarks/oheap.sh
@@ -174,8 +174,9 @@ EOF
 # This lets you figure out what the common types are, as well as the common
 # lengths.

-decode() {
-  for bin in _tmp/oheap/*.oheap; do
+decode-all() {
+  for bin in _tmp/oheap/*__oheap; do
+    echo $bin
    time _tmp/osh_demo $bin | wc -l
  done
 }
diff --git a/test/wild.sh b/test/wild.sh
@@ -670,6 +670,17 @@ analyze-noclobber() {
  wc -l $out
 }

+# Quick golden test.  Test that pretty-printing doesn't regress.
+golden-subset() {
+  $0 all esoteric
+}
+
+# Make a copy of the output for comparison.
+copy-golden-ast() {
+  local dest=${1:-_tmp/wild-gold}
+  find _tmp/wild/www/esoteric/ -name '*__ast.html' -a -printf '%p %P\n' \
+    | ~/git/tree-tools/bin/multi cp $dest
+}

 if test "$(basename $0)" = 'wild.sh'; then
  "$@"