Interleave parsing and execution in the main loop.

This is needed for:

- alias: runtime information is fed back into the parsing process
- when a shell scripts "changes languages" below 'exit'

Introduced a new file main_loop.py, which has Interactive() and Batch()
functions.  These call CommandParser.ParseOne() and
Executor.ExecuteAndCatch(), which are new interfaces.

Also move AST printing with -n into core/ui.py.  The main file is
greatly simplified.

NOTE: apparently this fixed a parse error!  In test/spec.sh
parse-errors.

Also:
- More notes about wild parse errors
- More tests for alias: inside subshell, eval, etc.

All unit tests and spec tests pass.  test/parse-errors.sh passes.

Still TODO:
- Get rid of ParseWholeFile
- There is a test failure in test/gold.sh (that doesn't make any other
  test fail?)

bin/oil.py

@@ -55,26 +55,21 @@ def _tlog(msg):
 # Set in Modules/main.c.
 HAVE_READLINE = os.getenv('_HAVE_READLINE') != ''
 
-from asdl import format as fmt
-from asdl import encode
-
 from osh import word_parse  # for tracing
 from osh import cmd_parse  # for tracing
 
-from osh import ast_lib
 from osh import parse_lib
 
 from core import alloc
 from core import args
 from core import builtin
 from core import cmd_exec
-from osh.meta import Id
 from core import legacy
 from core import lexer  # for tracing
+from core import main_loop
 from core import process
 from core import reader
 from core import state
-from core import word
 from core import word_eval
 from core import ui
 from core import util
@@ -95,54 +90,6 @@ def _tlog(msg):
 _tlog('after imports')
 
 
-def InteractiveLoop(opts, ex, c_parser, arena):
-  if opts.show_ast:
-    ast_f = fmt.DetectConsoleOutput(sys.stdout)
-  else:
-    ast_f = None
-
-  status = 0
-  while True:
-    # NOTE: We no longer need to catch KeyboardInterrupt here, because we
-    # handle SIGINT ourselves.
-    w = c_parser.Peek()
-
-    c_id = word.CommandId(w)
-    if c_id == Id.Op_Newline:
-      #print('nothing to execute')
-      pass
-    elif c_id == Id.Eof_Real:
-      print('EOF')
-      break
-    else:
-      try:
-        node = c_parser.ParseCommandLine()
-      except util.ParseError as e:
-        ui.PrettyPrintError(e, arena)
-
-        c_parser.Reset()
-        c_parser.ResetInputObjects()
-        continue
-      assert node is not None
-
-      if ast_f:
-        ast_lib.PrettyPrint(node)
-
-      status, is_control_flow = ex.ExecuteAndCatch(node)
-      if is_control_flow:  # exit or return
-        break
-
-      if opts.print_status:
-        print('STATUS', repr(status))
-
-    # Reset internal newline state.  NOTE: It would actually be correct to
-    # reinitialize all objects (except Env) on every iteration.
-    c_parser.Reset()
-    c_parser.ResetInputObjects()
-
-  return status
-
-
 # bash --noprofile --norc uses 'bash-4.3$ '
 OSH_PS1 = 'osh$ '
 
@@ -239,17 +186,9 @@ def OshMain(argv0, argv, login_shell):
       rc_line_reader = reader.FileLineReader(f, arena)
       _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena, aliases)
       try:
-        rc_node = rc_c_parser.ParseWholeFile()
-        if not rc_node:
-          err = rc_c_parser.Error()
-          ui.PrintErrorStack(err, arena)
-          return 2  # parse error is code 2
+        status = main_loop.Batch(opts, ex, rc_c_parser, arena)
       finally:
         arena.PopSource()
-
-    status = ex.Execute(rc_node)
-    #print('oilrc:', status, cflow, file=sys.stderr)
-    # Ignore bad status?
   except IOError as e:
     if e.errno != errno.ENOENT:
       raise
@@ -297,34 +236,17 @@ def OshMain(argv0, argv, login_shell):
       completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
                       status_out, ev)
 
-    return InteractiveLoop(opts, ex, c_parser, arena)
+    return main_loop.Interactive(opts, ex, c_parser, arena)
   else:
     # Parse the whole thing up front
     #print('Parsing file')
 
-    _tlog('ParseWholeFile')
-    # TODO: Do I need ParseAndEvalLoop?  How is it different than
-    # InteractiveLoop?
-
-    # TODO: InteractiveLoop above should use the same form of error handling.
-    # ParseWholeFile vs. ParseCommandLine
-    # I think ParseCommandLine should just be a loop
-    try:
-      node = c_parser.ParseWholeFile()
-    except util.ParseError as e:
-      ui.PrettyPrintError(e, arena, sys.stderr)
-      return 2
-    assert node is not None
-
-    do_exec = True
     if opts.fix:
-      osh2oil.PrintAsOil(arena, node, opts.debug_spans)
-      do_exec = False
+      #osh2oil.PrintAsOil(arena, node, opts.debug_spans)
+      raise AssertionError
     if opts.parse_and_print_arena:
       osh2oil.PrintArena(arena)
-      do_exec = False
-    if exec_opts.noexec:
-      do_exec = False
+      raise AssertionError
 
     # Do this after parsing the entire file.  There could be another option to
     # do it before exiting runtime?
@@ -340,59 +262,28 @@ def OshMain(argv0, argv, login_shell):
         log('Wrote %s to %s (--parser-mem-dump)', input_path,
             opts.parser_mem_dump)
 
-    # -n prints AST, --show-ast prints and executes
-    if exec_opts.noexec or opts.show_ast:
-      if opts.ast_format == 'none':
-        print('AST not printed.', file=sys.stderr)
-      elif opts.ast_format == 'oheap':
-        # TODO: Make this a separate flag?
-        if sys.stdout.isatty():
-          raise RuntimeError('ERROR: Not dumping binary data to a TTY.')
-        f = sys.stdout
-
-        enc = encode.Params()
-        out = encode.BinOutput(f)
-        encode.EncodeRoot(node, enc, out)
-
-      else:  # text output
-        f = sys.stdout
-
-        if opts.ast_format in ('text', 'abbrev-text'):
-          ast_f = fmt.DetectConsoleOutput(f)
-        elif opts.ast_format in ('html', 'abbrev-html'):
-          ast_f = fmt.HtmlOutput(f)
-        else:
-          raise AssertionError
-        abbrev_hook = (
-            ast_lib.AbbreviateNodes if 'abbrev-' in opts.ast_format else None)
-        tree = fmt.MakeTree(node, abbrev_hook=abbrev_hook)
-        ast_f.FileHeader()
-        fmt.PrintTree(tree, ast_f)
-        ast_f.FileFooter()
-        ast_f.write('\n')
-
-      #util.log("Execution skipped because 'noexec' is on ")
-      status = 0
-
-    if do_exec:
-      _tlog('Execute(node)')
-      status = ex.ExecuteAndRunExitTrap(node)
-      # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
-      # don't because they call sys.exit().
-      if opts.runtime_mem_dump:
-        # This might be superstition, but we want to let the value stabilize
-        # after parsing.  bash -c 'cat /proc/$$/status' gives different results
-        # with a sleep.
-        time.sleep(0.001)
-        input_path = '/proc/%d/status' % os.getpid()
-        with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
-          contents = f.read()
-          f2.write(contents)
-          log('Wrote %s to %s (--runtime-mem-dump)', input_path,
-              opts.runtime_mem_dump)
+    nodes_out = [] if exec_opts.noexec else None
 
-    else:
-      status = 0
+    _tlog('Execute(node)')
+    #status = ex.ExecuteAndRunExitTrap(node)
+    status = main_loop.Batch(opts, ex, c_parser, arena, nodes_out=nodes_out)
+
+    if nodes_out is not None:
+      ui.PrintAst(nodes_out, opts)
+
+    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
+    # don't because they call sys.exit().
+    if opts.runtime_mem_dump:
+      # This might be superstition, but we want to let the value stabilize
+      # after parsing.  bash -c 'cat /proc/$$/status' gives different results
+      # with a sleep.
+      time.sleep(0.001)
+      input_path = '/proc/%d/status' % os.getpid()
+      with open(input_path) as f, open(opts.runtime_mem_dump, 'w') as f2:
+        contents = f.read()
+        f2.write(contents)
+        log('Wrote %s to %s (--runtime-mem-dump)', input_path,
+            opts.runtime_mem_dump)
 
   return status
 

core/cmd_exec.py

@@ -1187,9 +1187,33 @@ def _ExecuteList(self, children):
       status = self._Execute(child)  # last status wins
     return status
 
+  def LastStatus(self):
+    """For main_loop.py to determine the exit code of the shell itself."""
+    return self.mem.last_status
+
   def ExecuteAndCatch(self, node, fork_external=True):
-    """Used directly by the interactive loop."""
+    """Execute a subprogram, handling _ControlFlow and fatal exceptions.
+
+    Args:
+      node: LST subtree
+      fork_external: whether external commands require forking
+
+    Returns:
+      TODO: use enum 'why' instead of the 2 booleans
+
+    Used by main_loop.py.
+
+    Also:
+    - SubProgramThunk for pipelines, subshell, command sub, process sub
+    - TODO: Signals besides EXIT trap
+
+    Most other clients call _Execute():
+    - _Source() for source builtin
+    - _Eval() for eval builtin
+    - RunFunc() for function call
+    """
     is_control_flow = False
+    is_fatal = False
     try:
       status = self._Execute(node, fork_external=fork_external)
     except _ControlFlow as e:
@@ -1201,59 +1225,40 @@ def ExecuteAndCatch(self, node, fork_external=True):
         raise  # Invalid
     except util.FatalRuntimeError as e:
       ui.PrettyPrintError(e, self.arena)
-      #print('osh failed: %s' % e.UserErrorString(), file=sys.stderr)
+      is_fatal = True
       status = e.exit_status if e.exit_status is not None else 1
       # TODO: dump self.mem if requested.  Maybe speify with OIL_DUMP_PREFIX.
 
-    # Other exceptions: SystemExit for sys.exit()
-    return status, is_control_flow
+    self.mem.last_status = status
+    return is_control_flow, is_fatal
 
+  # NOTE: Is this only used by tests?
   def Execute(self, node, fork_external=True):
     """Execute a subprogram, handling _ControlFlow and fatal exceptions.
 
-    This is just like ExecuteAndCatch, but we don't return is_control_flow.
-
-    Callers:
-    - SubProgramThunk for pipelines, subshell, command sub, process sub
-    - .oilrc
-    - _TrapThunk
-    - Interactive loop
-    - main program
-
-    Most other clients call _Execute():
-    - _Source() for source builtin
-    - _Eval() for eval builtin
-    - RunFunc() for function call
-
-    Args:
-      node: LST subtree
-      fork_external: whether external commands require forking
+    This is just like ExecuteAndCatch(), but we return a status.
 
     Returns:
       status: numeric exit code
     """
-    # Ignore is_control_flow
-    status, _ = self.ExecuteAndCatch(node, fork_external=fork_external)
-    return status
-
-  def ExecuteAndRunExitTrap(self, node):
-    """For the top level program, called by bin/oil.py."""
-    status = self.Execute(node)
-    # NOTE: 'exit 1' is ControlFlow and gets here, but subshell/commandsub
-    # don't because they call sys.exit().
+    self.ExecuteAndCatch(node, fork_external=fork_external)
+    return self.LastStatus()
 
-    # NOTE: --runtime-mem-dump runs in a similar place.
-
-    #log('-- EXIT pid %d', os.getpid())
-    #import traceback
-    #traceback.print_stack()
+  def MaybeRunExitTrap(self):
+    """If an EXIT trap exists, run it.
+    
+    Returns:
+      Whether we should use the status of the handler.
 
-    # NOTE: The trap handler itself can call exit!
+      This is odd behavior, but all bash/dash/mksh seem to agree on it.
+      See cases 11 and 12 in builtin-trap.test.sh.
+    """
     handler = self.traps.get('EXIT')
     if handler:
-      self.Execute(handler.node)
-
-    return status
+      is_control_flow, is_fatal = self.ExecuteAndCatch(handler.node)
+      return is_control_flow  # explicit exit/return in the trap handler!
+    else:
+      return False  # nothing run, don't use its status
 
   def RunCommandSub(self, node):
     p = self._MakeProcess(node,