Start integrating ASDL into the codebase.

- Move osh.asdl to the right directory.
- Add osh/ast.py to expose the classes to Python
- Import it from osh/bool_parse.py, but not hooked up yet
- Fix Python imports to be relative to the repo root.
- Adjust the line count script for ASDL code

Python's module system is very finicky:

Having a module named asdl.py inside a package named asdl is very bad,
and not sure if it's mentioned here:

http://python-notes.curiousefficiency.org/en/latest/python_concepts/import_traps.html

asdl/arith_ast.py

@@ -6,12 +6,12 @@
 import os
 import sys
 
-import asdl
-import py_meta
+from asdl import asdl_parse
+from asdl import py_meta
 
 this_dir = os.path.dirname(os.path.abspath(sys.argv[0]))
 schema_path = os.path.join(this_dir, 'arith.asdl')
 
-module = asdl.parse(schema_path)
+module = asdl_parse.parse(schema_path)
 root = sys.modules[__name__]
 py_meta.MakeTypes(module, root)

asdl/arith_ast_test.py

@@ -5,8 +5,8 @@
 
 import unittest
 
-import arith_ast  # module under test
-import py_meta
+from asdl import arith_ast  # module under test
+from asdl import py_meta
 
 
 class ArithAstTest(unittest.TestCase):

asdl/arith_parse.py

@@ -5,9 +5,9 @@
 
 import sys
 
-import tdop
-from tdop import Node, CompositeNode
-import arith_ast
+from asdl import tdop
+from asdl.tdop import Node, CompositeNode
+from asdl import arith_ast
 
 #
 # Null Denotation -- token that takes nothing on the left

asdl/arith_parse_test.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
-import tdop
-import arith_parse
+from asdl import tdop
+from asdl import arith_parse
 
 
 def _assertParseError(make_parser, s, error_substring=''):

asdl/asdl_demo.py

@@ -1,13 +1,13 @@
 #!/usr/bin/env python3
 """
-asdl_tool.py
+asdl_demo.py
 """
 
 import sys
-import asdl
-import arith_parse
-import py_meta
-import encode
+from asdl import asdl_parse
+from asdl import arith_parse
+from asdl import py_meta
+from asdl import encode
 
 
 def main(argv):
@@ -19,7 +19,7 @@ def main(argv):
   if action == 'py':
     schema_path = argv[2]
 
-    module = asdl.parse(schema_path)
+    module = asdl_parse.parse(schema_path)
     root = sys.modules[__name__]
     py_meta.MakeTypes(module, root)
     print(dir(root))

asdl/encode.py

@@ -5,8 +5,9 @@
 
 import sys
 
-import asdl
-import py_meta
+from asdl import asdl_parse
+from asdl import py_meta
+asdl = asdl_parse  # ALIAS
 
 _DEFAULT_ALIGNMENT = 4
 

asdl/encode_test.py

@@ -5,7 +5,7 @@
 
 import unittest
 
-import encode  # module under test
+from asdl import encode  # module under test
 
 
 class EncoderTest(unittest.TestCase):

asdl/gen_cpp.py

@@ -26,9 +26,11 @@
 
 import sys
 
-import asdl
-import py_meta
-import encode
+from asdl import asdl_parse
+from asdl import py_meta
+from asdl import encode
+
+asdl = asdl_parse
 
 TABSIZE = 2
 MAX_COL = 80

asdl/py_meta.py

@@ -29,7 +29,8 @@
 import sys
 import pprint
 
-import asdl
+from asdl import asdl_parse
+asdl = asdl_parse  # ALIAS for nodes
 
 
 def _CheckType(value, expected_desc):
@@ -109,6 +110,10 @@ class CompoundObj(Obj):
   FIELDS = []  # ordered list of field names
   DESCRIPTOR_LOOKUP = {}  # field name: (asdl.Type | int | str)
 
+  # Always set for constructor types, which are subclasses of sum types.  Never
+  # set for product types.
+  tag = None
+
   def __init__(self, *args, **kwargs):
     # The user must specify ALL required fields or NONE.
     self._assigned = {f: False for f in self.FIELDS}
@@ -157,7 +162,8 @@ def __setattr__(self, name, value):
       self.__dict__[name] = value
       return
     desc = self.DESCRIPTOR_LOOKUP[name]
-    if not _CheckType(value, desc):
+    if False:  # Disable type checking for now
+    #if not _CheckType(value, desc):
       raise AssertionError("Field %r should be of type %s, got %r" %
                            (name, desc, value))
 

asdl/py_meta_test.py

@@ -6,7 +6,7 @@
 import re
 import unittest
 
-import py_meta  # module under test
+from asdl import py_meta  # module under test
 
 class AsdlTest(unittest.TestCase):
   pass

asdl/run.sh

@@ -9,6 +9,8 @@ set -o nounset
 set -o pipefail
 set -o errexit
 
+export PYTHONPATH=.
+
 # Run unit tests.
 unit() {
   asdl/arith_ast_test.py
@@ -60,7 +62,7 @@ arith-both() {
 }
 
 osh-both() {
-  py-cpp asdl/osh.asdl
+  py-cpp osh/osh.asdl
 }
 
 #
@@ -199,21 +201,4 @@ compare-opts() {
   opt-stats
 }
 
-count() {
-  wc -l asdl/{asdl,py_meta,gen_cpp,encode}.py 
-  echo
-
-  wc -l asdl/{py_meta,encode}_test.py
-  echo
-
-  wc -l asdl/arith_parse*.py asdl/tdop.py asdl/arith_ast.py asdl/asdl_demo.py
-  echo
-
-  wc -l asdl/*.cc 
-  echo
-
-  wc -l asdl/*.asdl
-  echo
-}
-
 "$@"

core/expr_eval.py

@@ -275,6 +275,7 @@ def _EvalCompoundWord(self, word, do_glob=False):
     return s
 
   def _Eval(self, node):
+    # TODO: Switch on node.tag.
     if node.id == Id.Word_Compound:
       s = self._EvalCompoundWord(node)
       return bool(s)

count.sh

@@ -22,6 +22,22 @@ all() {
   wc -l tests/*.test.sh | sort --numeric
   echo
 
+  echo 'ASDL'
+  wc -l asdl/{asdl_parse,py_meta,gen_cpp,encode}.py 
+  echo
+
+  wc -l asdl/{py_meta,encode}_test.py
+  echo
+
+  #wc -l asdl/arith_parse*.py asdl/tdop.py asdl/arith_ast.py asdl/asdl_demo.py
+  #echo
+
+  #wc -l asdl/*.cc 
+  #echo
+
+  #wc -l asdl/*.asdl
+  #echo
+
   echo 'OIL UNIT TESTS'
   wc -l {osh,core}/*_test.py | sort --numeric
   echo

osh/ast.py

@@ -0,0 +1,19 @@
+#!/usr/bin/env python3
+"""
+osh/ast.py
+
+We parse osh.asdl and dynamically create classes on this module.
+"""
+
+import os
+import sys
+
+from asdl import py_meta
+from asdl import asdl_parse
+
+bin_dir = os.path.dirname(os.path.abspath(sys.argv[0]))  # ~/git/oil/bin
+schema_path = os.path.join(bin_dir, '../osh/osh.asdl')  # ~/git/oil/osh
+
+module = asdl_parse.parse(schema_path)
+root = sys.modules[__name__]
+py_meta.MakeTypes(module, root)

osh/bool_parse.py

@@ -54,6 +54,8 @@
 
 import sys
 
+from osh import ast
+
 from core import base
 from core.id_kind import Id, Kind, LookupKind, IdName
 
@@ -192,6 +194,7 @@ def ParseNegatedFactor(self):
       if not self._Next(): return None
       child = self.ParseFactor()
       return UnaryExprNode(Id.KW_Bang, child)
+      #return ast.LogicalNot(child)
     else:
       return self.ParseFactor()
 

asdl/osh.asdl → osh/osh.asdl

@@ -57,7 +57,8 @@ module osh
   | FuncCall(arith_expr func, arith_expr* args)
 
   bool_expr =
-    BoolBinary(word left, word right)
+    WordTest(word w)  -- e.g. [[ myword ]]
+  | BoolBinary(word left, word right)
   | BoolUnary(word child)
   | LogicalNot(bool_expr b)
   | LogicalAnd(bool_expr left, bool_expr right)