Progress toward migrating away from asdl/py_meta.py.

Not quite done yet. Equality and pretty printing of still need to be handled by asdl/gen_python.py. - Profile running abuild -h. These will eventually be osh runtime benchmarks. - Fix osh/lex_test.py
oilshell · Nov 28, 2017 · 8ba80cb6896034da3eac72dc9a1ced4667fddfce · 8ba80cb
1 parent 268155d
commit 8ba80cb6896034da3eac72dc9a1ced4667fddfce
Unified Split

Showing with 96 additions and 25 deletions.

+43 −0 asdl/asdl_base.py

+11 −23 asdl/gen_python.py

+10 −1 benchmarks/pytrace.sh

+31 −0 core/test_lib.py

+1 −1 osh/lex_test.py
diff --git a/asdl/asdl_base.py b/asdl/asdl_base.py
@@ -0,0 +1,43 @@
+#!/usr/bin/python
+"""
+asdl_base.py
+
+Base classes for generated code.
+"""
+import io
+
+from core import util
+
+
+# Copied from py_meta
+class Obj(object):
+  # NOTE: We're using CAPS for these static fields, since they are constant at
+  # runtime after metaprogramming.
+  DESCRIPTOR = None  # Used for type checking
+
+
+class SimpleObj(Obj):
+  def __init__(self, enum_id, name):
+    self.enum_id = enum_id
+    self.name = name
+
+  def __repr__(self):
+    return '<%s %s %s>' % (self.__class__.__name__, self.name, self.enum_id)
+
+
+class CompoundObj(Obj):
+  FIELDS = []  # ordered list of field names, overriden
+  DESCRIPTOR_LOOKUP = {}  # field name: (asdl.Sum | asdl.Product | ...)
+
+"""
+  def __repr__(self):
+    # Breaking circular dependency, gah.
+    from asdl import format as fmt
+
+    ast_f = fmt.TextOutput(util.Buffer())  # No color by default.
+    ast_f = fmt.AnsiOutput(io.StringIO())
+    tree = fmt.MakeTree(self)
+    fmt.PrintTree(tree, ast_f)
+    s, _ = ast_f.GetRaw()
+    return s
+    """
diff --git a/asdl/gen_python.py b/asdl/gen_python.py
@@ -21,7 +21,7 @@ class GenClassesVisitor(gen_cpp.AsdlVisitor):
  #     but also in _Init?

  def VisitSimpleSum(self, sum, name, depth):
-    self.Emit('class %s_e(SimpleObj):' % name, depth)
+    self.Emit('class %s_e(asdl_base.SimpleObj):' % name, depth)
    self.Emit('  pass', depth)
    self.Emit('', depth)

@@ -49,11 +49,14 @@ def _GenClass(self, desc, name, super_name, depth, tag_num=None,
    # rid of FIELDS?  Or you can just make it an alias.
    # FIELDS = self.__slots__.
    self.Emit('  FIELDS = %s' % quoted_fields, depth)
-    # Dummy values.
-    self.Emit('  DESCRIPTOR_LOOKUP = {}', depth)
-    self.Emit('  DESCRIPTOR = None', depth)

    self.Emit('  __slots__ = %s' % quoted_fields, depth)
+
+    # TODO: 
+    # py_meta.MakeTypes and py_meta._MakeFieldDescriptors fill
+    # DESCRIPTOR_LOOKUP, which is used for pretty printing.
+    lookup = {}
+    self.Emit('  DESCRIPTOR_LOOKUP = %r' % lookup, depth)
    self.Emit('', depth)

    args = ', '.join('%s=None' % f.name for f in desc.fields)
@@ -102,7 +105,7 @@ def VisitCompoundSum(self, sum, name, depth):
      self.Emit('  %s = %d' % (variant.name, i + 1), depth)
    self.Emit('', depth)

-    self.Emit('class %s(object):' % name, depth)
+    self.Emit('class %s(asdl_base.CompoundObj):' % name, depth)
    self.Emit('  pass', depth)
    self.Emit('', depth)

@@ -113,7 +116,7 @@ def VisitCompoundSum(self, sum, name, depth):
      self.VisitConstructor(t, super_name, tag_num, depth)

  def VisitProduct(self, product, name, depth):
-    self._GenClass(product, name, 'object', depth)
+    self._GenClass(product, name, 'asdl_base.CompoundObj', depth)

  def EmitFooter(self):
    pass
@@ -127,24 +130,9 @@ def main(argv):

  f = sys.stdout

-  # For const.NO_INTEGER
  f.write("""\
-from asdl import const
-
-# Copied from py_meta
-class Obj(object):
-  # NOTE: We're using CAPS for these static fields, since they are constant at
-  # runtime after metaprogramming.
-  DESCRIPTOR = None  # Used for type checking
-
-class SimpleObj(Obj):
-  def __init__(self, enum_id, name):
-    self.enum_id = enum_id
-    self.name = name
-
-  def __repr__(self):
-    return '<%s %s %s>' % (self.__class__.__name__, self.name, self.enum_id)
-
+from asdl import const  # For const.NO_INTEGER
+from asdl import asdl_base
 """)

  v = GenClassesVisitor(f)
diff --git a/benchmarks/pytrace.sh b/benchmarks/pytrace.sh
@@ -24,11 +24,17 @@ readonly -a OSH_PARSE=(bin/oil.py osh --ast-format none -n)
 time-run-abuild() { time "${RUN_ABUILD[@]}"; }
 time-parse-abuild() { time "${OSH_PARSE[@]}" $ABUILD; }

+_cprofile() {
+  local out=$1
+  shift
+  time python -m cProfile -o $out "$@"
+}
+
 # 3.8 seconds.  So less than 2x overhead.
 cprofile-osh-parse() {
  local in=${1:-$ABUILD}
  local out=${2:-abuild.cprofile}
-  time python -m cProfile -o $out "${OSH_PARSE[@]}" $in
+  _cprofile $out "${OSH_PARSE[@]}" $in
  ls -l $out
 }

@@ -38,6 +44,9 @@ cprofile-parse-abuild() {
 cprofile-parse-configure() {
  cprofile-osh-parse benchmarks/testdata/configure _tmp/configure.cprofile
 }
+cprofile-run-abuild() {
+  _cprofile _tmp/abuild-run.cprofile "${RUN_ABUILD[@]}"
+}

 # Yeah I understand from this why Chrome Tracing / Flame Graphs are better.
 # This format doesn't respect the stack!
diff --git a/core/test_lib.py b/core/test_lib.py
@@ -24,6 +24,37 @@ def TokenWordsEqual(left, right):
  #return left == right


+
+def AsdlEqual(left, right):
+  if isinstance(left, (int, str, bool)):
+    return left == right
+
+  if isinstance(left, list):
+    if len(left) != len(right):
+      return False
+    for a, b in zip(left, right):
+      if not AsdlEqual(a, b):
+        return False
+    return True
+
+  if isinstance(other, asdl.CompoundObj):
+    if left.tag != right.tag:
+      return False
+
+    for name in left.FIELDS:
+      # Special case: we are not testing locations right now.
+      if name == 'span_id':
+        continue
+      a = getattr(left, name)
+      b = getattr(right, name)
+      if not AsdlEqual(left, right):
+        return False
+
+    return True
+
+  raise AssertionError
+
+
 def MakeArena(source_name):
  pool = alloc.Pool()
  arena = pool.NewArena()
diff --git a/osh/lex_test.py b/osh/lex_test.py
@@ -206,7 +206,7 @@ def testLookAhead(self):
    l = LineLexer(parse_lib._MakeMatcher(), 'foo  bar')
    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'foo'), l.Read(lex_mode_e.OUTER))
-    self.assertEqual(
+    self.assertTokensEqual(
        ast.token(Id.Lit_Chars, 'bar'), l.LookAhead(lex_mode_e.OUTER))

    # No lookahead; using the cursor!