osh/meta.py is a new module with metaprogrammed data structures.

Explicit data is better for managing dependencies, e.g. between
osh/ast_.py and core/id_kind.py.  Those modules will become pure
libraries.

Also:
- RedirType is now an ASDL type; not util.Enum.
- Move pretty printing into its own module, to make dependencies clearer.
- Temporarily move REDIR_TYPE.

asdl/asdl_demo.py

@@ -11,7 +11,7 @@
from asdl import encode
from asdl import format as fmt

from core.id_kind import Id
from osh.meta import Id
from core import util

log = util.log

bin/oil.py

@@ -63,13 +63,14 @@ def _tlog(msg):
from osh import cmd_parse  # for tracing

from osh import ast_ as ast
from osh import ast_lib
from osh import parse_lib

from core import alloc
from core import args
from core import builtin
from core import cmd_exec
from core.id_kind import Id
from osh.meta import Id
from core import legacy
from core import lexer  # for tracing
from core import process
@@ -127,7 +128,7 @@ def InteractiveLoop(opts, ex, c_parser, w_parser, line_reader):
        raise RuntimeError('failed parse: %s' % c_parser.Error())

      if ast_f:
        ast.PrettyPrint(node)
        ast_lib.PrettyPrint(node)

      status = ex.Execute(node)

core/braces.py

@@ -17,7 +17,7 @@

import sys

from core.id_kind import Id
from osh.meta import Id
from osh import ast_ as ast

word_part_e = ast.word_part_e

core/cmd_exec.py

@@ -34,7 +34,7 @@
from core import ui
from core import util
from core import builtin
from core.id_kind import Id, RedirType, REDIR_TYPE, REDIR_DEFAULT_FD
from osh.meta import Id, REDIR_DEFAULT_FD, _InitRedirType
from core import process
from core import runtime
from core import state
@@ -43,6 +43,9 @@
from osh import ast_ as ast
from osh import parse_lib


REDIR_TYPE = _InitRedirType()

try:
  import libc  # for fnmatch
except ImportError:
@@ -54,6 +57,7 @@
redir_e = ast.redir_e
lhs_expr_e = ast.lhs_expr_e
assign_op_e = ast.assign_op_e
redir_type_e = ast.redir_type_e
lex_mode_e = ast.lex_mode_e

value_e = runtime.value_e
@@ -424,7 +428,7 @@ def _EvalRedirect(self, n):
    if n.tag == redir_e.Redir:
      redir_type = REDIR_TYPE[n.op_id]  # could be static in the LST?

      if redir_type == RedirType.Path:
      if redir_type == redir_type_e.Path:
        # NOTE: no globbing.  You can write to a file called '*.py'.
        val = self.word_ev.EvalWordToString(n.arg_word)
        if val.tag != value_e.Str:  # TODO: This error never fires
@@ -438,7 +442,7 @@ def _EvalRedirect(self, n):

        return runtime.PathRedirect(n.op_id, fd, filename)

      elif redir_type == RedirType.Desc:  # e.g. 1>&2
      elif redir_type == redir_type_e.Desc:  # e.g. 1>&2
        val = self.word_ev.EvalWordToString(n.arg_word)
        if val.tag != value_e.Str:  # TODO: This error never fires
          util.error("Redirect descriptor should be a string, got %s", val)
@@ -456,7 +460,7 @@ def _EvalRedirect(self, n):

        return runtime.DescRedirect(n.op_id, fd, target_fd)

      elif redir_type == RedirType.Here:  # here word
      elif redir_type == redir_type_e.Here:  # here word
        # TODO: decay should be controlled by an option
        val = self.word_ev.EvalWordToString(n.arg_word, decay=True)
        if val.tag != value_e.Str:   # TODO: This error never fires

core/cmd_exec_test.py

@@ -13,7 +13,7 @@

from core import cmd_exec  # module under test
from core.cmd_exec import *
from core.id_kind import Id
from osh.meta import Id
from core import completion
from core import legacy
from core import word_eval

core/completion_test.py

@@ -20,7 +20,7 @@
from core import test_lib
from core import word_eval
from core import ui
from core.id_kind import Id
from osh.meta import Id

from osh import ast_ as ast
from osh import parse_lib

core/expr_eval.py

@@ -17,7 +17,8 @@
except ImportError:
  from benchmarks import fake_libc as libc

from core.id_kind import BOOL_OPS, OperandType, Id
from osh.meta import BOOL_OPS, Id
from core.id_kind import OperandType
from core import util
from core import runtime

core/id_kind.py

@@ -15,60 +15,13 @@
from core import util


_ID_TO_KIND = {}  # int -> Kind

def LookupKind(id_):
  return _ID_TO_KIND[id_.enum_value]


_ID_NAMES = {}  # int -> string

def IdName(id_):
  return _ID_NAMES[id_.enum_value]



# Keep one instance of each Id, to save memory and enable comparison by
# OBJECT IDENTITY.
# Do NOT create any any more instances of them!  Always used IdInstance().

# TODO: Fold this into ASDL, which will enforce this?

_ID_INSTANCES = {}  # int -> Id

def IdInstance(i):
  return _ID_INSTANCES[i]


class Id(object):
  """Token and op type.

  The evaluator must consider all Ids.

  NOTE: We add a bunch of class attributes that are INSTANCES of this class,
  e.g. Id.Lit_Chars.
  """
  def __init__(self, enum_value):
    self.enum_value = enum_value

  def __repr__(self):
    return IdName(self)


class Kind(object):
  """A coarser version of Id, used to make parsing decisions."""

  # TODO: The Kind type should be folded into ASDL.  It can't print itself,
  # which is inconsistent with Id.
  pass


class IdSpec(object):
  """Identifiers that form the "spine" of the shell program representation."""

  def __init__(self, token_names, instance_lookup, kind_lookup, bool_ops):
    self.id_enum = Id
    self.kind_enum = Kind
  def __init__(self, id_enum, kind_enum,
               token_names, instance_lookup, kind_lookup, bool_ops):
    self.id_enum = id_enum
    self.kind_enum = kind_enum
    self.token_names = token_names  # integer -> string Id
    self.instance_lookup = instance_lookup
    self.kind_lookup = kind_lookup  # Id -> Kind
@@ -92,7 +45,8 @@ def _AddId(self, token_name, kind=None):
      kind: override autoassignment.  For AddBoolBinaryForBuiltin
    """
    self.token_index += 1  # leave out 0 I guess?
    id_val = Id(self.token_index)
    # The ONLY place that Id() is instantiated.
    id_val = self.id_enum(self.token_index)
    setattr(self.id_enum, token_name, id_val)

    t = self.token_index
@@ -104,6 +58,7 @@ def _AddId(self, token_name, kind=None):
    return id_val

  def _AddKind(self, kind_name):
    # TODO: Should be instantiated or folded into ASDL.
    setattr(self.kind_enum, kind_name, self.kind_index)
    #util.log('%s = %d', kind_name, self.kind_index)
    self.kind_index += 1
@@ -408,13 +363,6 @@ def _AddKinds(spec):
  ])


# Id -> OperandType
BOOL_OPS = {}  # type: dict

TEST_UNARY_LOOKUP = {}
TEST_BINARY_LOOKUP = {}
TEST_OTHER_LOOKUP = {}

# Shared between [[ and test/[.
_UNARY_STR_CHARS = 'zn'  # -z -n
_UNARY_OTHER_CHARS = 'otvR'  # -o is overloaded 
@@ -431,7 +379,7 @@ def _Dash(strs):
  return [(s, '-' + s) for s in strs]


def _AddBoolKinds(spec):
def _AddBoolKinds(spec, Id):
  spec.AddBoolKind('BoolUnary', {
      OperandType.Str: _Dash(list(_UNARY_STR_CHARS)),
      OperandType.Other: _Dash(list(_UNARY_OTHER_CHARS)),
@@ -456,7 +404,8 @@ def _AddBoolKinds(spec):
  spec.AddBoolOp(Id.Redir_Great, OperandType.Str)


def _SetupTestBuiltin(id_spec, unary_lookup, binary_lookup, other_lookup):
def _SetupTestBuiltin(Id, Kind, id_spec,
                      unary_lookup, binary_lookup, other_lookup):
  """Setup tokens for test/[.

  Similar to _AddBoolKinds above.  Differences:
@@ -490,64 +439,3 @@ def _SetupTestBuiltin(id_spec, unary_lookup, binary_lookup, other_lookup):
  other_lookup[')'] = Id.Op_RParen

  other_lookup[']'] = Id.Arith_RBracket  # For closing ]


#
# Instantiate the spec
#


ID_SPEC = IdSpec(_ID_NAMES, _ID_INSTANCES, _ID_TO_KIND, BOOL_OPS)

_AddKinds(ID_SPEC)
_AddBoolKinds(ID_SPEC)  # must come second
_SetupTestBuiltin(ID_SPEC, TEST_UNARY_LOOKUP, TEST_BINARY_LOOKUP,
                  TEST_OTHER_LOOKUP)


# Debug
_kind_sizes = ID_SPEC.kind_sizes


#
# Redirect Tables associated with IDs
#
# These might be osh specific.
#

REDIR_DEFAULT_FD = {
    # filename
    Id.Redir_Less: 0,  # cat <input.txt means cat 0<input.txt
    Id.Redir_Great: 1,
    Id.Redir_DGreat: 1,
    Id.Redir_Clobber: 1,
    Id.Redir_LessGreat: 1,  # TODO: What does echo <>foo do?

    # descriptor
    Id.Redir_GreatAnd: 1,  # echo >&2  means echo 1>&2
    Id.Redir_LessAnd: 0,   # echo <&3 means echo 0<&3, I think

    Id.Redir_TLess: 0,  # here word

    # here docs included
    Id.Redir_DLess: 0,
    Id.Redir_DLessDash: 0,
}

RedirType = util.Enum('RedirType', 'Path Desc Here'.split())

REDIR_TYPE = {
    # filename
    Id.Redir_Less: RedirType.Path,
    Id.Redir_Great: RedirType.Path,
    Id.Redir_DGreat: RedirType.Path,
    Id.Redir_Clobber: RedirType.Path,
    Id.Redir_LessGreat: RedirType.Path,

    # descriptor
    Id.Redir_GreatAnd: RedirType.Desc,
    Id.Redir_LessAnd: RedirType.Desc,

    Id.Redir_TLess: RedirType.Here,  # here word
    # note: here docs aren't included
}

core/id_kind_gen.py

@@ -13,7 +13,7 @@
import sys

from asdl.gen_cpp import FormatLines
from core.id_kind import Id, Kind, LookupKind
from osh.meta import Id, Kind, LookupKind


def Emit(s, f, depth=0):

core/id_kind_test.py

@@ -13,7 +13,10 @@
import unittest

from core import id_kind
from core.id_kind import Id, IdName, Kind, LookupKind
from osh.meta import (
    Id, IdName, IdInstance,
    Kind, LookupKind,
    ID_SPEC, BOOL_OPS, _ID_NAMES, _kind_sizes)

from osh import ast_ as ast

@@ -42,7 +45,7 @@ def testTokens(self):

    print('Number of Kinds:', num_kinds)
    # 233 out of 256 tokens now
    print('Number of IDs:', len(id_kind._ID_NAMES))
    print('Number of IDs:', len(_ID_NAMES))

    # Make sure we're not exporting too much
    print(dir(id_kind))
@@ -61,8 +64,8 @@ def testTokens(self):
    self.assertEqual(Kind.BoolBinary, LookupKind(t.id))

  def testEquality(self):
    left = id_kind.IdInstance(198)
    right = id_kind.IdInstance(198)
    left = IdInstance(198)
    right = IdInstance(198)
    print(left, right)
    print(left == right)
    self.assertEqual(left, right)
@@ -71,17 +74,17 @@ def testLexerPairs(self):
    def MakeLookup(p):
      return dict((pat, tok) for _, pat, tok in p)

    lookup = MakeLookup(id_kind.ID_SPEC.LexerPairs(Kind.BoolUnary))
    lookup = MakeLookup(ID_SPEC.LexerPairs(Kind.BoolUnary))
    print(lookup)
    self.assertEqual(Id.BoolUnary_e, lookup['-e'])
    self.assertEqual(Id.BoolUnary_z, lookup['-z'])

    lookup2 = MakeLookup(id_kind.ID_SPEC.LexerPairs(Kind.BoolBinary))
    lookup2 = MakeLookup(ID_SPEC.LexerPairs(Kind.BoolBinary))
    self.assertEqual(Id.BoolBinary_eq, lookup2['-eq'])
    #print(lookup2)

  def testPrintStats(self):
    k = id_kind._kind_sizes
    k = _kind_sizes
    print('STATS: %d tokens in %d groups: %s' % (sum(k), len(k), k))
    # Thinking about switching
    big = [i for i in k if i > 8]
@@ -91,8 +94,8 @@ def testPrintStats(self):


def PrintBoolTable():
  for i, arg_type in id_kind.BOOL_OPS.items():
    row = (id_kind.IdName(i), arg_type)
  for i, arg_type in BOOL_OPS.items():
    row = (IdName(i), arg_type)
    print('\t'.join(str(c) for c in row))

core/lexer.py

@@ -13,7 +13,7 @@

from asdl import const
from core import util
from core.id_kind import Id
from osh.meta import Id
from osh import ast_ as ast

log = util.log

core/lexer_test.py

@@ -5,7 +5,7 @@

import unittest

from core.id_kind import Id
from osh.meta import Id
from osh.lex import LEXER_DEF

from osh import ast_ as ast