Interleave parsing and execution in the main loop.

This is needed for:

- alias: runtime information is fed back into the parsing process
- when a shell scripts "changes languages" below 'exit'

Introduced a new file main_loop.py, which has Interactive() and Batch()
functions.  These call CommandParser.ParseOne() and
Executor.ExecuteAndCatch(), which are new interfaces.

Also move AST printing with -n into core/ui.py.  The main file is
greatly simplified.

NOTE: apparently this fixed a parse error!  In test/spec.sh
parse-errors.

Also:
- More notes about wild parse errors
- More tests for alias: inside subshell, eval, etc.

All unit tests and spec tests pass.  test/parse-errors.sh passes.

Still TODO:
- Get rid of ParseWholeFile
- There is a test failure in test/gold.sh (that doesn't make any other
  test fail?)

bin/oil.py

@@ -55,26 +55,21 @@ def _tlog(msg):
# Set in Modules/main.c.
HAVE_READLINE = os.getenv('_HAVE_READLINE') != ''

from asdl import format as fmt
from asdl import encode

from osh import word_parse  # for tracing
from osh import cmd_parse  # for tracing

from osh import ast_lib
from osh import parse_lib

from core import alloc
from core import args
from core import builtin
from core import cmd_exec
from osh.meta import Id
from core import legacy
from core import lexer  # for tracing
from core import main_loop
from core import process
from core import reader
from core import state
from core import word
from core import word_eval
from core import ui
from core import util
@@ -95,54 +90,6 @@ def _tlog(msg):
_tlog('after imports')


def InteractiveLoop(opts, ex, c_parser, arena):
  if opts.show_ast:
    ast_f = fmt.DetectConsoleOutput(sys.stdout)
  else:
    ast_f = None

  status = 0
  while True:
    # NOTE: We no longer need to catch KeyboardInterrupt here, because we
    # handle SIGINT ourselves.
    w = c_parser.Peek()

    c_id = word.CommandId(w)
    if c_id == Id.Op_Newline:
      #print('nothing to execute')
      pass
    elif c_id == Id.Eof_Real:
      print('EOF')
      break
    else:
      try:
        node = c_parser.ParseCommandLine()
      except util.ParseError as e:
        ui.PrettyPrintError(e, arena)

        c_parser.Reset()
        c_parser.ResetInputObjects()
        continue
      assert node is not None

      if ast_f:
        ast_lib.PrettyPrint(node)

      status, is_control_flow = ex.ExecuteAndCatch(node)
      if is_control_flow:  # exit or return
        break

      if opts.print_status:
        print('STATUS', repr(status))

    # Reset internal newline state.  NOTE: It would actually be correct to
    # reinitialize all objects (except Env) on every iteration.
    c_parser.Reset()
    c_parser.ResetInputObjects()

  return status


# bash --noprofile --norc uses 'bash-4.3$ '
OSH_PS1 = 'osh$ '

@@ -239,17 +186,9 @@ def OshMain(argv0, argv, login_shell):
      rc_line_reader = reader.FileLineReader(f, arena)
      _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena, aliases)
      try:
        rc_node = rc_c_parser.ParseWholeFile()
        if not rc_node:
          err = rc_c_parser.Error()
          ui.PrintErrorStack(err, arena)
          return 2  # parse error is code 2
        status = main_loop.Batch(opts, ex, rc_c_parser, arena)
      finally:
        arena.PopSource()

    status = ex.Execute(rc_node)
    #print('oilrc:', status, cflow, file=sys.stderr)
    # Ignore bad status?
  except IOError as e:
    if e.errno != errno.ENOENT:
      raise
@@ -297,34 +236,17 @@ def OshMain(argv0, argv, login_shell):
      completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
                      status_out, ev)

    return InteractiveLoop(opts, ex, c_parser, arena)
    return main_loop.Interactive(opts, ex, c_parser, arena)
  else:
    # Parse the whole thing up front
    #print('Parsing file')

    _tlog('ParseWholeFile')
    # TODO: Do I need ParseAndEvalLoop?  How is it different than
    # InteractiveLoop?

    # TODO: InteractiveLoop above should use the same form of error handling.
    # ParseWholeFile vs. ParseCommandLine
    # I think ParseCommandLine should just be a loop
    try:
      node = c_parser.ParseWholeFile()
    except util.ParseError as e:
      ui.PrettyPrintError(e, arena, sys.stderr)
      return 2
    assert node is not None

    do_exec = True
    if opts.fix:
      osh2oil.PrintAsOil(arena, node, opts.debug_spans)
      do_exec = False
      #osh2oil.PrintAsOil(arena, node, opts.debug_spans)
      raise AssertionError
    if opts.parse_and_print_arena:
      osh2oil.PrintArena(arena)
      do_exec = False
    if exec_opts.noexec:
      do_exec = False
      raise AssertionError

    # Do this after parsing the entire file.  There could be another option to
    # do it before exiting runtime?
@@ -340,59 +262,28 @@ def OshMain(argv0, argv, login_shell):
        log('Wrote %s to %s (--parser-mem-dump)', input_path,
            opts.parser_mem_dump)

    # -n prints AST, --show-ast prints and executes
    if exec_opts.noexec or opts.show_ast:
      if opts.ast_format == 'none':
        print('AST not printed.', file=sys.stderr)
      elif opts.ast_format == 'oheap':
        # TODO: Make this a separate flag?
        if sys.stdout.isatty():
          raise RuntimeError('ERROR: Not dumping binary data to a TTY.')
        f = sys.stdout

        enc = encode.Params()
        out = encode.BinOutput(f)
        encode.EncodeRoot(node, enc, out)

      else:  # text output
        f = sys.stdout

        if opts.ast_format in ('text', 'abbrev-text'):
          ast_f = fmt.DetectConsoleOutput(f)
        elif opts.ast_format in ('html', 'abbrev-html'):
          ast_f = fmt.HtmlOutput(f)
        else:
          raise AssertionError
        abbrev_hook = (
            ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None)
        tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
        ast_f.FileHeader()
        fmt.PrintTree(tree, ast_f)
        ast_f.FileFooter()
        ast_f.write('\n')

      #util.log("Execution skipped because 'noexec' is on ")
      status = 0

    if do_exec:
      _tlog('Execute(node)')
      status = ex.ExecuteAndRunExitTrap(node)
      # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
      # don't because they call sys.exit().
      if opts.runtime_mem_dump:
        # This might be superstition, but we want to let the value stabilize
        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
        # with a sleep.
        time.sleep(0.001)
        input_path = '/proc/%d/status' % os.getpid()
        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
          contents = f.read()
          f2.write(contents)
          log('Wrote %s to %s (--runtime-mem-dump)', input_path,
              opts.runtime_mem_dump)
    nodes_out = [] if exec_opts.noexec else None

    else:
      status = 0
    _tlog('Execute(node)')
    #status = ex.ExecuteAndRunExitTrap(node)
    status = main_loop.Batch(opts, ex, c_parser, arena, nodes_out=nodes_out)

    if nodes_out is not None:
      ui.PrintAst(nodes_out, opts)

    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    if opts.runtime_mem_dump:
      # This might be superstition, but we want to let the value stabilize
      # after parsing.  bash -c 'cat /proc/$$/status' gives different results
      # with a sleep.
      time.sleep(0.001)
      input_path = '/proc/%d/status' % os.getpid()
      with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
        contents = f.read()
        f2.write(contents)
        log('Wrote %s to %s (--runtime-mem-dump)', input_path,
            opts.runtime_mem_dump)

  return status

core/cmd_exec.py

@@ -1187,9 +1187,33 @@ def _ExecuteList(self, children):
      status = self._Execute(child)  # last status wins
    return status

  def LastStatus(self):
    """For main_loop.py to determine the exit code of the shell itself."""
    return self.mem.last_status

  def ExecuteAndCatch(self, node, fork_external=True):
    """Used directly by the interactive loop."""
    """Execute a subprogram, handling _ControlFlow and fatal exceptions.

    Args:
      node: LST subtree
      fork_external: whether external commands require forking

    Returns:
      TODO: use enum 'why' instead of the 2 booleans

    Used by main_loop.py.

    Also:
    - SubProgramThunk for pipelines, subshell, command sub, process sub
    - TODO: Signals besides EXIT trap

    Most other clients call _Execute():
    - _Source() for source builtin
    - _Eval() for eval builtin
    - RunFunc() for function call
    """
    is_control_flow = False
    is_fatal = False
    try:
      status = self._Execute(node, fork_external=fork_external)
    except _ControlFlow as e:
@@ -1201,59 +1225,40 @@ def ExecuteAndCatch(self, node, fork_external=True):
        raise  # Invalid
    except util.FatalRuntimeError as e:
      ui.PrettyPrintError(e, self.arena)
      #print('osh failed: %s' % e.UserErrorString(), file=sys.stderr)
      is_fatal = True
      status = e.exit_status if e.exit_status is not None else 1
      # TODO: dump self.mem if requested.  Maybe speify with OIL_DUMP_PREFIX.

    # Other exceptions: SystemExit for sys.exit()
    return status, is_control_flow
    self.mem.last_status = status
    return is_control_flow, is_fatal

  # NOTE: Is this only used by tests?
  def Execute(self, node, fork_external=True):
    """Execute a subprogram, handling _ControlFlow and fatal exceptions.

    This is just like ExecuteAndCatch, but we don't return is_control_flow.

    Callers:
    - SubProgramThunk for pipelines, subshell, command sub, process sub
    - .oilrc
    - _TrapThunk
    - Interactive loop
    - main program

    Most other clients call _Execute():
    - _Source() for source builtin
    - _Eval() for eval builtin
    - RunFunc() for function call

    Args:
      node: LST subtree
      fork_external: whether external commands require forking
    This is just like ExecuteAndCatch(), but we return a status.

    Returns:
      status: numeric exit code
    """
    # Ignore is_control_flow
    status, _ = self.ExecuteAndCatch(node, fork_external=fork_external)
    return status

  def ExecuteAndRunExitTrap(self, node):
    """For the top level program, called by bin/oil.py."""
    status = self.Execute(node)
    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
    # don't because they call sys.exit().
    self.ExecuteAndCatch(node, fork_external=fork_external)
    return self.LastStatus()

    # NOTE: --runtime-mem-dump runs in a similar place.

    #log('-- EXIT pid %d', os.getpid())
    #import traceback
    #traceback.print_stack()
  def MaybeRunExitTrap(self):
    """If an EXIT trap exists, run it.
    
    Returns:
      Whether we should use the status of the handler.

    # NOTE: The trap handler itself can call exit!
      This is odd behavior, but all bash/dash/mksh seem to agree on it.
      See cases 11 and 12 in builtin-trap.test.sh.
    """
    handler = self.traps.get('EXIT')
    if handler:
      self.Execute(handler.node)

    return status
      is_control_flow, is_fatal = self.ExecuteAndCatch(handler.node)
      return is_control_flow  # explicit exit/return in the trap handler!
    else:
      return False  # nothing run, don't use its status

  def RunCommandSub(self, node):
    p = self._MakeProcess(node,