Fix up the arithmetic oheap demo.

The OSH oheap demo still doesn't work. - Tighten up error message around encoding simple optional types. - Fix up arith_ast_test.py in preparation for ASDL API changes.
oilshell · Nov 25, 2017 · e6c31c361e04952df0fdeaf73822b50337e167d2 · e6c31c3
1 parent 64305c1
commit e6c31c361e04952df0fdeaf73822b50337e167d2
diff --git a/asdl/arith.asdl b/asdl/arith.asdl
@@ -1,6 +1,15 @@
 module arith {
+  -- Simple Sum Type
  op_id = Plus | Minus | Star

+  -- Compound Sum Type
+  cflow =
+    Break
+  | Continue
+  | Return(int status)
+
+  -- Product Type
+
  -- why does source_location have to be lower case?  It's not a constructor?
  -- it has no tag?
  source_location = (string path, int line, int col, int length)
@@ -32,5 +41,7 @@ module arith {
  | LogicalNot(bool_expr b)
  | LogicalAnd(bool_expr left, bool_expr right)
  | LogicalOr(bool_expr left, bool_expr right)
+
+
 }

diff --git a/asdl/arith_ast_test.py b/asdl/arith_ast_test.py
@@ -10,6 +10,7 @@
 from asdl import format as fmt
 from asdl import py_meta
 from asdl import encode
+from asdl import asdl_


 ArithVar = arith_ast.ArithVar
@@ -21,6 +22,9 @@
 source_location = arith_ast.source_location
 op_id_e = arith_ast.op_id_e

+cflow_e = arith_ast.cflow_e
+#cflow_t = arith_ast.cflow_t
+

 class ArithAstTest(unittest.TestCase):

@@ -58,7 +62,33 @@ def testExtraFields(self):
    # What about product types?
    #print(v.xspans)

-  def testTypes(self):
+  def testEncode(self):
+    obj = arith_ast.Const(99)
+    print('Encoding into binary:')
+    print(obj)
+
+    enc = encode.Params()
+    f = io.BytesIO()
+    out = encode.BinOutput(f)
+    encode.EncodeRoot(obj, enc, out)
+    e = f.getvalue()
+
+    #print(repr(e))
+    #print(e[0:4], e[4:8], e[8:])
+
+    # Header is OHP version 1
+    self.assertEqual(b'OHP\x01', e[0:4])
+
+    self.assertEqual(b'\x04', e[4:5])  # alignment 4
+
+    # TODO: Fix after spids
+    return
+    self.assertEqual(b'\x02\x00\x00', e[5:8])  # root ref 2
+
+    self.assertEqual(b'\x01', e[8:9])  # tag 1 is const
+    self.assertEqual(b'\x63\x00\x00', e[9:12])  # 0x63 = 99
+
+  def testConstructorType(self):

    print(ArithVar)
    print('FIELDS', ArithVar.FIELDS)
@@ -92,7 +122,10 @@ def testTypes(self):
    else:
      raise AssertionError("Should have failed")

-    #n5 = ArithVar(None)
+  def testProductType(self):
+    print
+    print '-- PRODUCT --'
+    print

    s = source_location()
    s.path = 'hi'
@@ -101,14 +134,42 @@ def testTypes(self):
    s.length = 3
    print(s)

-    c = Const(66)
-    print(c)
-    # Test out hierarchy
-    assert isinstance(c, Const)
-    assert isinstance(c, arith_expr)
+    assert isinstance(s.DESCRIPTOR, asdl_.Product)
+
+    # Implementation detail for dynamic type checking
+    assert isinstance(s, py_meta.CompoundObj)
+
+  def testSimpleSumType(self):
+    # TODO: Should be op_id_i.Plus -- instance
+    # Should be op_id_s.Plus
+
+    print
+    print '-- SIMPLE SUM --'
+    print
+
+    o = op_id_e.Plus
+    assert isinstance(o, py_meta.SimpleObj)
+
+    # Implementation detail for dynamic type checking
+    assert isinstance(o.DESCRIPTOR, asdl_.Sum)
+
+  def testCompoundSumType(self):
+    print
+    print '-- COMPOUND SUM --'
+    print
+
+    # TODO: Should be cflow_t.Break() and cflow_i.Break
+    c = arith_ast.Break()
+    assert isinstance(c, arith_ast.Break)
+    assert isinstance(c, arith_ast.cflow)
    assert isinstance(c, py_meta.CompoundObj)

-    #print(Const('invalid'))
+    # Implementation detail for dynamic type checking
+    assert isinstance(c.DESCRIPTOR, asdl_.Constructor), c.DESCRIPTOR
+
+  def testOtherTypes(self):
+    c = Const(66)
+    print(c)

    print(Slice(Const(1), Const(5), Const(2)))

@@ -132,32 +193,6 @@ def testTypes(self):
    self.assertEqual(arith_expr_e.Const, c.tag)
    self.assertEqual(arith_expr_e.ArithBinary, n.tag)

-  def testEncode(self):
-    obj = arith_ast.Const(99)
-    print('Encoding into binary:')
-    print(obj)
-
-    enc = encode.Params()
-    f = io.BytesIO()
-    out = encode.BinOutput(f)
-    encode.EncodeRoot(obj, enc, out)
-    e = f.getvalue()
-
-    #print(repr(e))
-    #print(e[0:4], e[4:8], e[8:])
-
-    # Header is OHP version 1
-    self.assertEqual(b'OHP\x01', e[0:4])
-
-    self.assertEqual(b'\x04', e[4:5])  # alignment 4
-
-    # TODO: Fix after spids
-    return
-    self.assertEqual(b'\x02\x00\x00', e[5:8])  # root ref 2
-
-    self.assertEqual(b'\x01', e[8:9])  # tag 1 is const
-    self.assertEqual(b'\x63\x00\x00', e[9:12])  # 0x63 = 99
-

 if __name__ == '__main__':
  unittest.main()
diff --git a/asdl/arith_parse.py b/asdl/arith_parse.py
@@ -10,6 +10,9 @@
 from asdl.tdop import Node, CompositeNode
 from asdl import arith_ast

+op_id = arith_ast.op_id_e  # TODO: Rename this back.
+
+
 #
 # Null Denotation -- token that takes nothing on the left
 #
@@ -102,14 +105,14 @@ def LeftTernary(p, token, left, bp):
 def LeftBinaryOp(p, token, left, rbp):
  """ Normal binary operator like 1+2 or 2*3, etc. """
  if token.val == '+':
-    op_id = arith_ast.op_id.Plus
+    op_id_ = op_id.Plus
  elif token.val == '-':
-    op_id = arith_ast.op_id.Minus
+    op_id_ = op_id.Minus
  elif token.val == '*':
-    op_id = arith_ast.op_id.Star
+    op_id_ = op_id.Star
  else:
    raise AssertionError(token.val)
-  return arith_ast.ArithBinary(op_id, left, p.ParseUntil(rbp))
+  return arith_ast.ArithBinary(op_id_, left, p.ParseUntil(rbp))


 def LeftAssign(p, token, left, rbp):
diff --git a/asdl/asdl_.py b/asdl/asdl_.py
@@ -41,13 +41,16 @@ def is_simple(sum):


 class StrType:
-  pass
+  def __repr__(self):
+    return '<Str>'

 class IntType:
-  pass
+  def __repr__(self):
+    return '<Int>'

 class BoolType:
-  pass
+  def __repr__(self):
+    return '<Bool>'


 # NOTE: We might want to change this to PrimitiveType(int tag).  Then
@@ -65,10 +68,16 @@ class ArrayType:
  def __init__(self, desc):
    self.desc = desc

+  def __repr__(self):
+    return '<Array %s>' % self.desc
+
 class MaybeType:
  def __init__(self, desc):
    self.desc = desc  # another descriptor

+  def __repr__(self):
+    return '<Maybe %s>' % self.desc
+
 class UserType:
  def __init__(self, typ):
    assert isinstance(typ, type), typ
diff --git a/asdl/asdl_demo.py b/asdl/asdl_demo.py
@@ -23,7 +23,8 @@ def main(argv):
  if action == 'py':
    schema_path = argv[2]

-    module = asdl.parse(schema_path)
+    with open(schema_path) as f:
+      module = asdl.parse(f)
    root = sys.modules[__name__]
    # NOTE: We shouldn't pass in app_types for arith.asdl, but this is just a
    # demo.
diff --git a/asdl/encode.py b/asdl/encode.py
@@ -63,6 +63,7 @@ def __init__(self, alignment=_DEFAULT_ALIGNMENT):
    # also I guess steuff like SimpleCommand
    self.index_width = 2  # 16 bits, e.g. max 64K entries in an array

+    # TODO: check for negative too
    self.max_int = 1 << (self.ref_width * 8)
    self.max_index = 1 << (self.index_width * 8)
    self.max_tag = 1 << (self.tag_width * 8)
@@ -233,10 +234,11 @@ def EncodeObj(obj, enc, out):
      elif isinstance(item_desc, asdl.Product):
        ok = True

-      # TODO: Fix this for span_id.  Need to extract a method.
+      # TODO: We can catch this case when parsing the schema rather than on
+      # encoding.
      if not ok:
        raise AssertionError(
-            "Currently not encoding simple optional types: %s" % field_val)
+            "Simple types shouldn't be optional: %s %s" % (name, desc))

      if field_val is None:
        enc.Ref(0, this_chunk)
diff --git a/asdl/osh_demo.cc b/asdl/osh_demo.cc
@@ -22,7 +22,7 @@ int main(int argc, char **argv) {
  }
  FILE *f = fopen(argv[1], "rb");
  if (!f) {
-    printf("Error opening %s", argv[1]);
+    fprintf(stderr, "Error opening %s\n", argv[1]);
    return 1;
  }
  fseek(f, 0, SEEK_END);
diff --git a/asdl/run.sh b/asdl/run.sh
@@ -51,17 +51,8 @@ py-cpp() {
 # Test specific schemas
 #

-arith-both() {
-  py-cpp asdl/arith.asdl
-}
-
-osh-both() {
-  py-cpp osh/osh.asdl
-}
-
-ovm-both() {
-  py-cpp ovm/ovm.asdl
-}
+arith-both() { py-cpp asdl/arith.asdl; }
+osh-both() { py-cpp osh/osh.asdl; }

 #
 # Native Code
@@ -118,11 +109,16 @@ arith-demo() {
  #$bin $data 
 }

+# TODO: How big is oheap vs. the virtual memory size?
+
 osh-demo() {
  local name=osh
  local data=_tmp/${name}.bin

-  # TODO: Call bin/osh --ast-output _tmp/a.oheap --ast-format oheap -c 'echo # hi'
+  # TODO: Fix bug -- this is only 3 bytes?
+  bin/osh -n --ast-format oheap -c 'echo hi  # comment' > $data
+
+  ls -l $data

  core/id_kind_gen.py cpp > _tmp/id_kind.h
  build-demo osh/osh.asdl
diff --git a/asdl/tdop.py b/asdl/tdop.py
@@ -184,7 +184,7 @@ def AtToken(self, token_type):
  def Next(self):
    """Move to the next token."""
    try:
-      t = self.lexer.__next__()
+      t = self.lexer.next()
    except StopIteration:
      t = EOF_TOKEN
    self.token = t