Move types that don't depend on Id to osh/types.asdl.

Also fix errors caught by flake8.

build/codegen.sh

@@ -30,7 +30,7 @@ source build/common.sh
# needed for the Makefile to properly crawl dependencies.
#
# _devbuild/gen/
#    osh-ast.h - lex_mode_e for now
#    osh-types.h - lex_mode_e for now
#    id_kind.h - id_e for now
#    osh-lex.re2c.c  
#    osh-lex.c  
@@ -51,8 +51,8 @@ install-re2c() {

re2c() { _deps/re2c-1.0.3/re2c "$@"; }

ast-gen() {
  PYTHONPATH=. osh/ast_gen.py "$@" > _devbuild/gen/osh-ast.h
types-gen() {
  PYTHONPATH=. osh/ast_gen.py "$@" > _devbuild/gen/osh-types.h
}

id-gen() {
@@ -84,7 +84,7 @@ ast-id-lex() {
  mkdir -p _devbuild/gen

  log "-- Generating AST, IDs, and lexer in _devbuild/gen"
  ast-gen
  types-gen
  id-gen
  osh-lex-gen
  osh-lex-gen-native

build/dev.sh

@@ -31,6 +31,13 @@ gen-help() {
  #build/doc.sh oil-quick-ref
}

gen-types-asdl() {
  local out=_devbuild/gen/types_asdl.py
  local import='from osh.meta import TYPES_TYPE_LOOKUP as TYPE_LOOKUP'
  PYTHONPATH=. asdl/gen_python.py osh/types.asdl "$import" > $out
  echo "Wrote $out"
}

gen-osh-asdl() {
  local out=_devbuild/gen/osh_asdl.py
  local import='from osh.meta import OSH_TYPE_LOOKUP as TYPE_LOOKUP'
@@ -94,6 +101,7 @@ minimal() {
  touch _devbuild/__init__.py  _devbuild/gen/__init__.py

  gen-help
  gen-types-asdl
  gen-osh-asdl
  gen-runtime-asdl
  pylibc

core/cmd_exec.py

@@ -35,11 +35,10 @@
from core import util
from core import builtin
from core import process
from osh.meta import runtime
from core import state
from core import word_compile

from osh.meta import ast, Id, REDIR_TYPE, REDIR_DEFAULT_FD
from osh.meta import ast, Id, REDIR_TYPE, REDIR_DEFAULT_FD, runtime, types
from osh import parse_lib

try:
@@ -49,12 +48,13 @@

EBuiltin = builtin.EBuiltin

lex_mode_e = types.lex_mode_e
redir_type_e = types.redir_type_e

command_e = ast.command_e
redir_e = ast.redir_e
lhs_expr_e = ast.lhs_expr_e
assign_op_e = ast.assign_op_e
redir_type_e = ast.redir_type_e
lex_mode_e = ast.lex_mode_e

value_e = runtime.value_e
scope_e = runtime.scope_e

core/tdop.py

@@ -7,13 +7,13 @@
from core import word
from core import util

from osh.meta import ast
from osh.meta import ast, types

p_die = util.p_die

arith_expr_e = ast.arith_expr_e
word_e = ast.word_e
lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e


def Assert(s, expected, tree):

core/test_builtin.py

@@ -6,14 +6,11 @@

import sys

from core import id_kind
from core import expr_eval
from osh.meta import runtime
from core import util

from osh import bool_parse
from osh.meta import ast
from osh.meta import Id
from osh.meta import ast, Id, runtime
from osh import meta

log = util.log

native/fastlex.c

@@ -8,7 +8,7 @@
#include <Python.h>

#include "id.h"
#include "osh-ast.h"
#include "osh-types.h"  // for lex_mode_e
// this is generated C code, but we want a single translation unit
#include "osh-lex.h"

native/fastlex_test.py

@@ -11,12 +11,11 @@

import unittest

from osh.meta import Id, IdInstance
from osh.meta import ast
from osh.meta import Id, IdInstance, types

import fastlex  # module under test

lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e


def MatchToken(lex_mode, line, start_pos):

osh/ast_gen.py

@@ -5,42 +5,21 @@

import sys

from osh.meta import ast, Id
from osh.meta import types, Id
from asdl import asdl_ as asdl
from asdl import gen_cpp

lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e


def main(argv):
  #print lex_mode_e
  #print dir(lex_mode_e)

  app_types = {'id': asdl.UserType(Id)}
  with open('osh/osh.asdl') as f:
  with open('osh/types.asdl') as f:
    asdl_module, _ = asdl.LoadSchema(f, app_types)

  # TODO: Generate C files for lex_mode_e, id_e, etc.

  # It already works for lex_mode.  Except it's C++.  Maybe write one that's C?
  #
  # C doesn't have namespaces!

  # lex_mode_e__NONE?
  # lex_mode_e__OUTER?
  # The re2c code will have a switch statement for that?
  #
  # Or maybe just lex_mode__NONE.
  #
  # And then it will output id__Lit_Chars ?
  # This is all generated so we can change it at any time.

  # Eventually you may also generate code to map Id -> Kind?  But right now you
  # just need Id for the lexer!

  #print asdl_module
  #print gen_cpp

  v = gen_cpp.CEnumVisitor(sys.stdout)
  v.VisitModule(asdl_module)

osh/bool_parse.py

@@ -32,18 +32,16 @@
BINARY_OP: -gt, -ot, ==, etc.
"""

from osh.meta import ast

from core import word
from osh.meta import Id, Kind, LookupKind
from core import util
from osh.meta import ast, Id, Kind, LookupKind, types

try:
  import libc  # for regex_parse
except ImportError:
  from benchmarks import fake_libc as libc

lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e
log = util.log

osh/bool_parse_test.py

@@ -11,15 +11,14 @@

import unittest

from osh.meta import Id
from core import test_lib

from osh.meta import ast
from osh.meta import ast, Id, types
from osh import parse_lib
from osh import bool_parse  # module under test

bool_expr_e = ast.bool_expr_e
lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e


def _ReadWords(w_parser):

osh/cmd_parse.py

@@ -14,18 +14,17 @@

from core import braces
from core import word
from osh.meta import Id, Kind
from core import util

from osh.meta import ast
from osh.meta import ast, Id, Kind, types
from osh.lex import VAR_NAME_RE
from osh.bool_parse import BoolParser

log = util.log
command_e = ast.command_e
word_e = ast.word_e
assign_op_e = ast.assign_op_e
lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e


class CommandParser(object):

osh/lex.py

@@ -105,9 +105,9 @@
from osh.meta import Id, Kind, ID_SPEC
from core.lexer import C, R

from osh.meta import ast
from osh.meta import types

lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e


# In oil, I hope to have these lexer modes:

osh/lex_test.py

@@ -5,15 +5,14 @@

import unittest

from osh.meta import Id, Kind, LookupKind
from core.lexer import CompileAll, LineLexer
from core import test_lib

from osh import parse_lib
from osh.meta import ast
from osh.meta import ast, Id, Kind, LookupKind, types
from osh.lex import LEXER_DEF

lex_mode_e = ast.lex_mode_e
lex_mode_e = types.lex_mode_e


def _InitLexer(s):

osh/meta.py

@@ -75,6 +75,27 @@ def IdInstance(i):
  return _ID_INSTANCES[i]


#
# Instantiate osh/types.asdl
#

f = util.GetResourceLoader().open('osh/types.asdl')
_asdl_module, _type_lookup = asdl.LoadSchema(f, {})  # no app_types

types = _AsdlModule()
if 0:
  py_meta.MakeTypes(_asdl_module, ast, _type_lookup)
else:
  # Exported for the generated code to use
  TYPES_TYPE_LOOKUP = _type_lookup

  # Get the types from elsewhere
  from _devbuild.gen import types_asdl
  py_meta.AssignTypes(types_asdl, types)

f.close()


# Id -> OperandType
BOOL_OPS = {}  # type: dict

@@ -105,7 +126,7 @@ def IdInstance(i):
APP_TYPES = {'id': asdl.UserType(Id)}

#
# Instantiate the AST
# Instantiate osh/osh.asdl
#

f = util.GetResourceLoader().open('osh/osh.asdl')
@@ -125,15 +146,15 @@ def IdInstance(i):
f.close()

#
# Instantiate runtime
# Instantiate core/runtime.asdl
#

f = util.GetResourceLoader().open('core/runtime.asdl')
_asdl_module, _type_lookup = asdl.LoadSchema(f, APP_TYPES)

runtime = _AsdlModule()
if 0:
  py_meta.MakeTypes(module, runtime, _type_lookup)
  py_meta.MakeTypes(_asdl_module, runtime, _type_lookup)
else:
  # Exported for the generated code to use
  RUNTIME_TYPE_LOOKUP = _type_lookup
@@ -144,7 +165,6 @@ def IdInstance(i):

f.close()


#
# Redirect Tables associated with IDs
#
@@ -170,7 +190,7 @@ def IdInstance(i):
    Id.Redir_DLessDash: 0,
}

redir_type_e = ast.redir_type_e
redir_type_e = types.redir_type_e

REDIR_TYPE = {
    # filename

osh/osh.asdl

@@ -20,8 +20,7 @@

-- Parsed but Not Implemented
-- * extended glob
-- * Process sub -- <() and >()
-- * <>, >| redirects
-- * <> &> redirects

-- TODO: Preserve these source differences:
-- * order of redirects: 'echo >out.txt hi'  vs echo hi >out.txt
@@ -198,8 +197,6 @@ module osh
    -- this is useful for type safety.
  | CommandList(command* children)

  and_or = DAmp | DPipe

  -- For now, using stderr_indices representation because it's more compact.
  -- |& in osh; |- in oil.
  -- pipe_op = Pipe | PipeAndStderr
@@ -225,26 +222,4 @@ module osh
  -- These types are not used in the LST.  But they could be statically
  -- derived from values in the LST.
  --

  -- Also invalid because of duplicate 'Path' -- to fix
  -- bool_operand_type = Undefined | Path | Int | Str | Other
  redir_type = Path | Desc | Here

  --
  -- APIs
  --

  -- Fifteen lexer modes for osh.
  -- Possible additional modes:
  --   nested backticks: echo `echo \`echo foo\` bar`
  lex_mode =
    NONE
  | COMMENT
  | OUTER
  | DBRACKET
  | SQ | DQ | DOLLAR_SQ
  | ARITH
  | EXTGLOB
  | VS_1 | VS_2 | VS_ARG_UNQ | VS_ARG_DQ
  | BASH_REGEX | BASH_REGEX_CHARS
}

osh/parse_lib.py

@@ -4,7 +4,6 @@

from core import lexer
from core import reader
from core import id_kind

from osh import lex
from osh import word_parse