Move the AST from osh.ast_ -> osh.meta.ast.

This untangles metaprogramming dependencies, e.g. between id_kind and
osh.asdl.

bin/oil.py

@@ -62,7 +62,7 @@ def _tlog(msg):
from osh import word_parse  # for tracing
from osh import cmd_parse  # for tracing

from osh import ast_ as ast
from osh.meta import ast
from osh import ast_lib
from osh import parse_lib

build/dev.sh

@@ -33,7 +33,7 @@ gen-help() {

gen-osh-asdl() {
  local out=_devbuild/gen/osh_asdl.py
  PYTHONPATH=. asdl/gen_python.py osh/osh.asdl 'osh.ast_' > $out
  PYTHONPATH=. asdl/gen_python.py osh/osh.asdl 'osh.meta' > $out
  echo "Wrote $out"
}

core/braces.py

@@ -18,7 +18,7 @@
import sys

from osh.meta import Id
from osh import ast_ as ast
from osh.meta import ast

word_part_e = ast.word_part_e
word_e = ast.word_e

core/braces_test.py

@@ -9,7 +9,7 @@

from core import braces  # module under test
from osh import word_parse_test
from osh import ast_ as ast
from osh.meta import ast

word_part_e = ast.word_part_e

core/cmd_exec.py

@@ -34,18 +34,15 @@
from core import ui
from core import util
from core import builtin
from osh.meta import Id, REDIR_DEFAULT_FD, _InitRedirType
from osh.meta import Id, REDIR_TYPE, REDIR_DEFAULT_FD, _InitRedirType
from core import process
from core import runtime
from core import state
from core import word_compile

from osh import ast_ as ast
from osh.meta import ast
from osh import parse_lib


REDIR_TYPE = _InitRedirType()

try:
  import libc  # for fnmatch
except ImportError:

core/cmd_exec_test.py

@@ -20,7 +20,7 @@
from core import process
from core import test_lib

from osh import ast_ as ast
from osh.meta import ast
from osh import parse_lib

core/completion.py

@@ -35,7 +35,7 @@
import time
import traceback

from osh import ast_ as ast
from osh.meta import ast
from osh import parse_lib
from core import alloc
from core import runtime

core/completion_test.py

@@ -22,7 +22,7 @@
from core import ui
from osh.meta import Id

from osh import ast_ as ast
from osh.meta import ast
from osh import parse_lib

assign_op_e = ast.assign_op_e

core/expr_eval.py

@@ -22,7 +22,7 @@
from core import util
from core import runtime

from osh import ast_ as ast
from osh.meta import ast

log = util.log
warn = util.warn

core/id_kind_gen.py

@@ -13,7 +13,7 @@
import sys

from asdl.gen_cpp import FormatLines
from osh.meta import Id, Kind, LookupKind
from osh.meta import Id, Kind, LookupKind, ID_SPEC


def Emit(s, f, depth=0):
@@ -87,8 +87,6 @@ def main(argv):
    raise RuntimeError('Action required')

  if action == 'c':
    # Simple list of defines
    from core.id_kind import ID_SPEC
    ids = list(ID_SPEC.token_names.iteritems())
    ids.sort(key=lambda pair: pair[0])  # Sort by ID
    for i, name in ids:

core/id_kind_test.py

@@ -18,7 +18,7 @@
    Kind, LookupKind,
    ID_SPEC, BOOL_OPS, _ID_NAMES, _kind_sizes)

from osh import ast_ as ast
from osh.meta import ast


class TokensTest(unittest.TestCase):

core/lexer.py

@@ -14,7 +14,7 @@
from asdl import const
from core import util
from osh.meta import Id
from osh import ast_ as ast
from osh.meta import ast

log = util.log

core/lexer_gen.py

@@ -9,6 +9,7 @@
import sre_constants

from osh import lex
from osh import meta


def PrintTree(re_tree, depth=2):
@@ -228,9 +229,7 @@ def TranslateLexer(lexer_def):
        re2_pat = TranslateRegex(pat)
      else:
        re2_pat = TranslateConstant(pat)
      # TODO: Remove this after debugging Id problem
      from core import id_kind
      id_name = id_kind.IdName(token_id)
      id_name = meta.IdName(token_id)
      print '      %-30s { *id = id__%s; break; }' % (re2_pat, id_name)

    # EARLY RETURN: Do NOT advance past the NUL terminator.

core/lexer_test.py

@@ -8,7 +8,7 @@
from osh.meta import Id
from osh.lex import LEXER_DEF

from osh import ast_ as ast
from osh.meta import ast


class TokenTest(unittest.TestCase):

core/process_test.py

@@ -8,7 +8,7 @@

from osh.meta import Id
from core import builtin
from osh import ast_ as ast
from osh.meta import ast

from core import process  # module under test
from core import runtime

core/state.py

@@ -19,7 +19,7 @@
from core import util
from osh.meta import Id

from osh import ast_ as ast
from osh.meta import ast

part_value_e = runtime.part_value_e
value_e = runtime.value_e

core/tdop.py

@@ -7,7 +7,7 @@
from core import word
from core import util

from osh import ast_ as ast
from osh.meta import ast

p_die = util.p_die

core/test_builtin.py

@@ -12,7 +12,7 @@
from core import util

from osh import bool_parse
from osh import ast_ as ast
from osh.meta import ast
from osh.meta import Id
from osh import meta

core/word.py

@@ -4,7 +4,7 @@

from osh.meta import Id, Kind, LookupKind
from core import util
from osh import ast_ as ast
from osh.meta import ast
from asdl import const

p_die = util.p_die

core/word_eval.py

@@ -14,7 +14,7 @@
from core import state
from core import word_compile
from core import util
from osh import ast_ as ast
from osh.meta import ast

part_value_e = runtime.part_value_e
value_e = runtime.value_e

native/fastlex_test.py

@@ -12,7 +12,7 @@
import unittest

from osh.meta import Id, IdInstance
from osh import ast_ as ast
from osh.meta import ast

import fastlex  # module under test

osh/arith_parse.py

@@ -7,7 +7,7 @@
from core import util
from osh.meta import Id
from core import word
from osh import ast_ as ast
from osh.meta import ast

p_die = util.p_die

osh/ast_.py

@@ -3,16 +3,11 @@
osh/ast_.py -- Parse osh.asdl and dynamically create classes on this module.
"""

import sys

from asdl import asdl_ as asdl
from asdl import py_meta

from core import util
from osh.meta import Id


def LoadSchema(f):
def LoadSchema(Id, f):
  """Parse an ASDL schema.  Used for code gen and metaprogramming."""
  app_types = {'id': asdl.UserType(Id)}

  asdl_module = asdl.parse(f)
@@ -22,20 +17,3 @@ def LoadSchema(f):

  type_lookup = asdl.ResolveTypes(asdl_module, app_types)
  return asdl_module, type_lookup


f = util.GetResourceLoader().open('osh/osh.asdl')
asdl_module, type_lookup = LoadSchema(f)

root = sys.modules[__name__]
if 0:
  py_meta.MakeTypes(asdl_module, root, type_lookup)
else:
  # Exported for the generated code to use
  TYPE_LOOKUP = type_lookup

  # Get the types from elsewhere
  from _devbuild.gen import osh_asdl
  py_meta.AssignTypes(osh_asdl, root)

f.close()

osh/ast_gen.py

@@ -5,7 +5,8 @@

import sys

from osh import ast_ as ast
from osh.meta import ast, Id
from osh import ast_
from asdl import gen_cpp

lex_mode_e = ast.lex_mode_e
@@ -16,7 +17,7 @@ def main(argv):
  #print dir(lex_mode_e)

  with open('osh/osh.asdl') as f:
    asdl_module, _ = ast.LoadSchema(f)
    asdl_module, _ = ast_.LoadSchema(Id, f)

  # TODO: Generate C files for lex_mode_e, id_e, etc.

osh/ast_lib.py

@@ -108,5 +108,3 @@ def PrettyPrint(node, f=sys.stdout):
  tree = fmt.MakeTree(node, AbbreviateNodes)
  fmt.PrintTree(tree, ast_f)
  f.write('\n')

osh/bool_parse.py

@@ -32,7 +32,7 @@
BINARY_OP: -gt, -ot, ==, etc.
"""

from osh import ast_ as ast
from osh.meta import ast

from core import word
from osh.meta import Id, Kind, LookupKind

osh/bool_parse_test.py

@@ -14,7 +14,7 @@
from osh.meta import Id
from core import test_lib

from osh import ast_ as ast
from osh.meta import ast
from osh import parse_lib
from osh import bool_parse  # module under test

osh/cmd_parse.py

@@ -17,7 +17,7 @@
from osh.meta import Id, Kind
from core import util

from osh import ast_ as ast
from osh.meta import ast
from osh.lex import VAR_NAME_RE
from osh.bool_parse import BoolParser

osh/cmd_parse_test.py

@@ -11,7 +11,7 @@
from core import word
from core import test_lib

from osh import ast_ as ast
from osh.meta import ast
from osh import ast_lib
from osh import parse_lib
from osh.cmd_parse import CommandParser  # module under test

osh/lex.py

@@ -105,7 +105,7 @@
from osh.meta import Id, Kind, ID_SPEC
from core.lexer import C, R

from osh import ast_ as ast
from osh.meta import ast

lex_mode_e = ast.lex_mode_e

osh/lex_test.py

@@ -10,7 +10,7 @@
from core import test_lib

from osh import parse_lib
from osh import ast_ as ast
from osh.meta import ast
from osh.lex import LEXER_DEF

lex_mode_e = ast.lex_mode_e