Introduce ParseContext and thread it throughout the code.

This is mainly motivated by 'alias'.  Aliases are unfortunately truly
global state.

It also breaks some longstanding circular imports between word_parse.py
and cmd_parse.py, which is nice.

All unit, spec, and gold tests pass.  Still need to do some cleanup.

bin/oil.py

@@ -161,12 +161,14 @@ def OshMain(argv0, argv, login_shell):
  builtin.SetExecOpts(exec_opts, opts.opt_changes)
  aliases = {}  # feedback between runtime and parser

  parse_ctx = parse_lib.ParseContext(arena, aliases)

  if opts.debug_file:
    util.DEBUG_FILE = fd_state.Open(opts.debug_file, mode='w')
    util.Debug('Debug file is %s', util.DEBUG_FILE)

  ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, readline,
                         completion, comp_lookup, exec_opts, arena, aliases)
                         completion, comp_lookup, exec_opts, parse_ctx)

  # NOTE: The rc file can contain both commands and functions... ideally we
  # would only want to save nodes/lines for the functions.
@@ -175,7 +177,7 @@ def OshMain(argv0, argv, login_shell):
    arena.PushSource(rc_path)
    with open(rc_path) as f:
      rc_line_reader = reader.FileLineReader(f, arena)
      _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena, aliases)
      _, rc_c_parser = parse_ctx.MakeParser(rc_line_reader)
      try:
        status = main_loop.Batch(ex, rc_c_parser, arena)
      finally:
@@ -215,7 +217,7 @@ def OshMain(argv0, argv, login_shell):

  # TODO: assert arena.NumSourcePaths() == 1
  # TODO: .rc file needs its own arena.
  w_parser, c_parser = parse_lib.MakeParser(line_reader, arena, aliases)
  w_parser, c_parser = parse_ctx.MakeParser(line_reader)

  if exec_opts.interactive:
    # NOTE: We're using a different evaluator here.  The completion system can
@@ -225,7 +227,7 @@ def OshMain(argv0, argv, login_shell):
      ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
      status_out = completion.StatusOutput(status_lines, exec_opts)
      completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
                      status_out, ev)
                      status_out, ev, parse_ctx)

    return main_loop.Interactive(opts, ex, c_parser, arena)

@@ -353,7 +355,8 @@ def OshCommandMain(argv):

  line_reader = reader.FileLineReader(f, arena)
  aliases = {}  # Dummy value; not respecting aliases!
  _, c_parser = parse_lib.MakeParser(line_reader, arena, aliases)
  parse_ctx = parse_lib.ParseContext(arena, aliases)
  _, c_parser = parse_ctx.MakeParser(line_reader)

  try:
    node = main_loop.ParseWholeFile(c_parser)

core/cmd_exec.py

@@ -41,7 +41,6 @@
from core import word_compile

from osh.meta import ast, Id, REDIR_ARG_TYPES, REDIR_DEFAULT_FD, runtime, types
from osh import parse_lib

try:
  import libc  # for fnmatch
@@ -107,7 +106,7 @@ class Executor(object):
  CompoundWord/WordPart.
  """
  def __init__(self, mem, fd_state, status_lines, funcs, readline, completion,
               comp_lookup, exec_opts, arena, aliases):
               comp_lookup, exec_opts, parse_ctx):
    """
    Args:
      mem: Mem instance for storing variables
@@ -130,7 +129,9 @@ def __init__(self, mem, fd_state, status_lines, funcs, readline, completion,
    # This is for shopt and set -o.  They are initialized by flags.
    self.exec_opts = exec_opts
    self.exec_opts.readline = readline
    self.arena = arena
    self.parse_ctx = parse_ctx
    self.arena = parse_ctx.arena
    self.aliases = parse_ctx.aliases  # alias name -> string

    self.splitter = legacy.SplitContext(self.mem)
    self.word_ev = word_eval.NormalWordEvaluator(
@@ -142,7 +143,6 @@ def __init__(self, mem, fd_state, status_lines, funcs, readline, completion,
    self.nodes_to_run = []  # list of nodes, appended to by signal handlers
    self.dir_stack = state.DirStack()

    self.aliases = aliases  # alias name -> string
    self.targets = []  # make syntax enters stuff here -- Target()
                       # metaprogramming or regular target syntax
                       # Whether argv[0] is make determines if it is executed
@@ -153,7 +153,7 @@ def __init__(self, mem, fd_state, status_lines, funcs, readline, completion,

    self.loop_level = 0  # for detecting bad top-level break/continue

    self.tracer = Tracer(exec_opts, mem, self.word_ev)
    self.tracer = Tracer(parse_ctx, exec_opts, mem, self.word_ev)
    self.check_command_sub_status = False  # a hack

  def _Complete(self, argv):
@@ -191,7 +191,7 @@ def _Eval(self, argv):
    # TODO: set -o sane-eval should change eval to take a single string.
    code_str = ' '.join(argv)
    line_reader = reader.StringLineReader(code_str, self.arena)
    _, c_parser = parse_lib.MakeParser(line_reader, self.arena, self.aliases)
    _, c_parser = self.parse_ctx.MakeParser(line_reader)
    return self._EvalHelper(c_parser, '<eval string>')

  def ParseTrapCode(self, code_str):
@@ -200,7 +200,7 @@ def ParseTrapCode(self, code_str):
      A node, or None if the code is invalid.
    """
    line_reader = reader.StringLineReader(code_str, self.arena)
    _, c_parser = parse_lib.MakeParser(line_reader, self.arena, self.aliases)
    _, c_parser = self.parse_ctx.MakeParser(line_reader)

    source_name = '<trap string>'
    self.arena.PushSource(source_name)
@@ -235,7 +235,7 @@ def _Source(self, argv):

    try:
      line_reader = reader.FileLineReader(f, self.arena)
      _, c_parser = parse_lib.MakeParser(line_reader, self.arena, self.aliases)
      _, c_parser = self.parse_ctx.MakeParser(line_reader)

      # A sourced module CAN have a new arguments array, but it always shares
      # the same variable scope as the caller.  The caller could be at either a
@@ -1441,13 +1441,14 @@ class Tracer(object):
    - set -x doesn't print line numbers!  OH but you can do that with
      PS4=$LINENO
  """
  def __init__(self, exec_opts, mem, word_ev):
  def __init__(self, parse_ctx, exec_opts, mem, word_ev):
    """
    Args:
      exec_opts: For xtrace setting
      mem: for retrieving PS4
      word_ev: for evaluating PS4
    """
    self.parse_ctx = parse_ctx
    self.exec_opts = exec_opts
    self.mem = mem
    self.word_ev = word_ev
@@ -1467,11 +1468,13 @@ def _EvalPS4(self):
    else:
      first_char, ps4 = '+', ' '  # default

    # NOTE: This cache is slightly broken because aliases are mutable!  I think
    # thati s more or less harmless though.
    try:
      ps4_word = self.parse_cache[ps4]
    except KeyError:
      # We have to parse this at runtime.  PS4 should usually remain constant.
      w_parser = parse_lib.MakeWordParserForPlugin(ps4, self.arena)
      w_parser = self.parse_ctx.MakeWordParserForPlugin(ps4, self.arena)

      try:
        ps4_word = w_parser.ReadPS()

core/cmd_exec_test.py

@@ -25,12 +25,10 @@


def InitCommandParser(code_str):
  from osh.word_parse import WordParser
  from osh.cmd_parse import CommandParser
  arena = test_lib.MakeArena('<cmd_exec_test.py>')
  parse_ctx = parse_lib.ParseContext(arena, {})
  line_reader, lexer = parse_lib.InitLexer(code_str, arena)
  w_parser = WordParser(lexer, line_reader)
  c_parser = CommandParser(w_parser, lexer, line_reader, arena)
  _, c_parser = parse_ctx.MakeParser(line_reader)
  return c_parser


@@ -43,11 +41,11 @@ def InitExecutor(arena=None):
  status_lines = None  # not needed for what we're testing
  funcs = {}
  comp_funcs = {}
  aliases = {}
  exec_opts = state.ExecOpts(mem)
  parse_ctx = parse_lib.ParseContext(arena, {})
  # For the tests, we do not use 'readline'.
  return cmd_exec.Executor(mem, fd_state, status_lines, funcs, None,
                           completion, comp_funcs, exec_opts, arena, aliases)
                           completion, comp_funcs, exec_opts, parse_ctx)


def InitEvaluator():

core/completion.py

@@ -36,7 +36,6 @@
import traceback

from osh.meta import ast, runtime
from osh import parse_lib
from core import alloc
from core import state
from core import ui
@@ -580,18 +579,19 @@ class RootCompleter(object):
  """
  Provide completion of a buffer according to the configured rules.
  """
  def __init__(self, pool, ev, comp_lookup, var_comp):
  def __init__(self, pool, ev, comp_lookup, var_comp, parse_ctx):
    self.pool = pool
    self.ev = ev
    self.comp_lookup = comp_lookup
    # This can happen in any position, with any command
    self.var_comp = var_comp
    self.parse_ctx = parse_ctx

    self.parser = DummyParser()  # TODO: remove

  def Matches(self, buf, status_out):
    arena = alloc.CompletionArena(self.pool)
    w_parser, c_parser = parse_lib.MakeParserForCompletion(buf, arena)
    w_parser, c_parser = self.parse_ctx.MakeParserForCompletion(buf, arena)
    comp_type, prefix, comp_words = _GetCompletionType(
        w_parser, c_parser, self.ev, status_out)

@@ -747,7 +747,7 @@ def Write(self, index, msg, *args):
      self.status_lines[index].Write(msg, *args)


def Init(pool, builtins, mem, funcs, comp_lookup, status_out, ev):
def Init(pool, builtins, mem, funcs, comp_lookup, status_out, ev, parse_ctx):

  aliases_action = WordsAction(['TODO:alias'])
  commands_action = ExternalCommandAction(mem)
@@ -774,7 +774,7 @@ def Init(pool, builtins, mem, funcs, comp_lookup, status_out, ev):
  comp_lookup.RegisterName('grep', C1)

  var_comp = VarAction(os.environ, mem)
  root_comp = RootCompleter(pool, ev, comp_lookup, var_comp)
  root_comp = RootCompleter(pool, ev, comp_lookup, var_comp, parse_ctx)

  complete_cb = ReadlineCompleter(root_comp, status_out)
  InitReadline(complete_cb)

core/completion_test.py

@@ -120,8 +120,10 @@ def testRootCompleter(self):
    ev = _MakeTestEvaluator()

    pool = alloc.Pool()
    arena = pool.NewArena()
    parse_ctx = parse_lib.ParseContext(arena, {})
    var_comp = V1
    r = completion.RootCompleter(pool, ev, comp_lookup, var_comp)
    r = completion.RootCompleter(pool, ev, comp_lookup, var_comp, parse_ctx)

    m = list(r.Matches('grep f', STATUS))
    self.assertEqual(['foo.py ', 'foo '], m)
@@ -162,7 +164,8 @@ def _MakeTestEvaluator():
def _TestGetCompletionType(buf):
  ev = _MakeTestEvaluator()
  arena = test_lib.MakeArena('<completion_test.py>')
  w_parser, c_parser = parse_lib.MakeParserForCompletion(buf, arena)
  parse_ctx = parse_lib.ParseContext(arena, {})
  w_parser, c_parser = parse_ctx.MakeParserForCompletion(buf, arena)
  print('---', buf)
  return completion._GetCompletionType(w_parser, c_parser, ev, STATUS)

osh/arith_parse_test.py

@@ -27,7 +27,8 @@

def ParseAndEval(code_str):
  arena = test_lib.MakeArena('<arith_parse_test.py>')
  w_parser, _ = parse_lib.MakeParserForCompletion(code_str, arena)
  parse_ctx = parse_lib.ParseContext(arena, {})
  w_parser, _ = parse_ctx.MakeParserForCompletion(code_str, arena)
  w_parser._Next(lex_mode_e.ARITH)  # Calling private method
  anode = w_parser._ReadArithExpr()  # need the right lex state?
  print('node:', anode)

osh/bool_parse_test.py

@@ -37,7 +37,8 @@ def _ReadWords(w_parser):
def _MakeParser(code_str):
  # NOTE: We need the extra ]] token
  arena = test_lib.MakeArena('<bool_parse_test.py>')
  w_parser, _ = parse_lib.MakeParserForCompletion(code_str + ' ]]', arena)
  parse_ctx = parse_lib.ParseContext(arena, {})
  w_parser, _ = parse_ctx.MakeParserForCompletion(code_str + ' ]]', arena)
  w_parser._Next(lex_mode_e.DBRACKET)  # for tests only
  p = bool_parse.BoolParser(w_parser)
  if not p._Next():

osh/cmd_parse.py

@@ -82,7 +82,7 @@ def _MakeLiteralHereLines(here_lines, arena):
  return [ast.LiteralPart(t) for t in tokens]


def _ParseHereDocBody(h, line_reader, arena):
def _ParseHereDocBody(parse_ctx, h, line_reader, arena):
  """Fill in attributes of a pending here doc node."""
  # "If any character in word is quoted, the delimiter shall be formed by
  # performing quote removal on word, and the here-document lines shall not
@@ -99,10 +99,8 @@ def _ParseHereDocBody(h, line_reader, arena):
    # LiteralPart for each line.
    h.stdin_parts = _MakeLiteralHereLines(here_lines, arena)
  else:
    from osh import parse_lib  # Avoid circular import

    line_reader = reader.HereDocLineReader(here_lines, arena)
    w_parser = parse_lib.MakeWordParserForHereDoc(line_reader, arena)
    w_parser = parse_ctx.MakeWordParserForHereDoc(line_reader)
    w_parser.ReadHereDocBody(h.stdin_parts)  # fills this in

  end_line_id, end_line, end_pos = last_line
@@ -259,7 +257,8 @@ class CommandParser(object):
    lexer: for lookahead in function def, PushHint of ()
    line_reader: for here doc
  """
  def __init__(self, w_parser, lexer_, line_reader, arena, aliases=None):
  def __init__(self, parse_ctx, w_parser, lexer_, line_reader, arena, aliases=None):
    self.parse_ctx = parse_ctx
    self.w_parser = w_parser  # for normal parsing
    self.lexer = lexer_  # for pushing hints, lookahead to (
    self.line_reader = line_reader  # for here docs
@@ -334,7 +333,7 @@ def _Peek(self):
      # count.
      if w.tag == word_e.TokenWord and w.token.id == Id.Op_Newline:
        for h in self.pending_here_docs:
          _ParseHereDocBody(h, self.line_reader, self.arena)
          _ParseHereDocBody(self.parse_ctx, h, self.line_reader, self.arena)
        del self.pending_here_docs[:]  # No .clear() until Python 3.3.

      self.cur_word = w
@@ -576,8 +575,7 @@ def _ExpandAliases(self, words, cur_aliases):

    # TODO: Change name back to VirtualLineReader?
    line_reader = reader.HereDocLineReader(line_info, self.arena)
    from osh import parse_lib
    _, cp = parse_lib.MakeParser(line_reader, self.arena, self.aliases)
    _, cp = self.parse_ctx.MakeParser(line_reader)

    try:
      node = cp.ParseCommand(cur_aliases=cur_aliases)

osh/cmd_parse_test.py

@@ -13,18 +13,15 @@
from osh.meta import ast, Id
from osh import ast_lib
from osh import parse_lib
from osh.cmd_parse import CommandParser  # module under test
from osh.word_parse import WordParser

command_e = ast.command_e


# TODO: Use parse_lib instead
def InitCommandParser(code_str):
  arena = test_lib.MakeArena('<cmd_parse_test.py>')
  parse_ctx = parse_lib.ParseContext(arena, {})
  line_reader, lexer = parse_lib.InitLexer(code_str, arena)
  w_parser = WordParser(lexer, line_reader)
  c_parser = CommandParser(w_parser, lexer, line_reader, arena)
  _, c_parser = parse_ctx.MakeParser(line_reader)
  return arena, c_parser  # arena is returned for printing errors