Clean up the interface to the command parser.

- word_parse.py uses c_parser.ParseCommandSub() for $() and ``
- source and eval use main_loop.Batch() to interleave parsing and
  execution (e.g. so they can have aliases defined.)
- ParseTrap uses main_loop.ParseWholeFile()
- oshc also uses main_loop.ParseWholeFile()

All unit, spec, gold, and osh2oil tests pass.

bin/oil.py

@@ -177,7 +177,7 @@ def OshMain(argv0, argv, login_shell):
      rc_line_reader = reader.FileLineReader(f, arena)
      _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena, aliases)
      try:
        status = main_loop.Batch(opts, ex, rc_c_parser, arena)
        status = main_loop.Batch(ex, rc_c_parser, arena)
      finally:
        arena.PopSource()
  except IOError as e:
@@ -250,7 +250,7 @@ def OshMain(argv0, argv, login_shell):

  _tlog('Execute(node)')
  #status = ex.ExecuteAndRunExitTrap(node)
  status = main_loop.Batch(opts, ex, c_parser, arena, nodes_out=nodes_out)
  status = main_loop.Batch(ex, c_parser, arena, nodes_out=nodes_out)

  if nodes_out is not None:
    ui.PrintAst(nodes_out, opts)
@@ -355,9 +355,8 @@ def OshCommandMain(argv):
  aliases = {}  # Dummy value; not respecting aliases!
  _, c_parser = parse_lib.MakeParser(line_reader, arena, aliases)

  # TODO: Should I use main_loop.Batch() here?
  try:
    node = c_parser.ParseWholeFile()
    node = main_loop.ParseWholeFile(c_parser)
  except util.ParseError as e:
    ui.PrettyPrintError(e, arena, sys.stderr)
    return 2

core/cmd_exec.py

@@ -26,17 +26,18 @@
from core import alloc
from core import args
from core import braces
from core import builtin
from core import expr_eval
from core import legacy
from core import main_loop
from core import process
from core import reader
from core import state
from core import test_builtin
from core import word
from core import word_eval
from core import ui
from core import util
from core import builtin
from core import process
from core import state
from core import word_compile

from osh.meta import ast, Id, REDIR_ARG_TYPES, REDIR_DEFAULT_FD, runtime, types
@@ -182,16 +183,7 @@ def _Complete(self, argv):
  def _EvalHelper(self, c_parser, source_name):
    self.arena.PushSource(source_name)
    try:
      try:
        node = c_parser.ParseWholeFile()
      except util.ParseError as e:
        util.error('Parse error in %r:', source_name)
        ui.PrettyPrintError(e, self.arena, sys.stderr)
        return 2

      status = self._Execute(node)
      return status

      return main_loop.Batch(self, c_parser, self.arena)
    finally:
      self.arena.PopSource()

@@ -215,7 +207,7 @@ def ParseTrapCode(self, code_str):

    try:
      try:
        node = c_parser.ParseWholeFile()
        node = main_loop.ParseWholeFile(c_parser)
      except util.ParseError as e:
        util.error('Parse error in %r:', source_name)
        ui.PrettyPrintError(e, self.arena, sys.stderr)

core/cmd_exec_test.py

@@ -106,34 +106,5 @@ def testVarOps(self):
    print(part_vals)


def ParseAndExecute(code_str):
  arena = test_lib.MakeArena('<shell_test.py>')

  # TODO: Unify with InitCommandParser above.
  from osh.word_parse import WordParser
  from osh.cmd_parse import CommandParser

  line_reader, lexer = parse_lib.InitLexer(code_str, arena)
  w_parser = WordParser(lexer, line_reader)
  c_parser = CommandParser(w_parser, lexer, line_reader, arena)

  node = c_parser.ParseWholeFile()
  if not node:
    raise AssertionError()

  print(node)
  ex = InitExecutor(arena)
  status = ex.Execute(node)

  # TODO: Can we capture output here?
  return status


class ExecutorTest(unittest.TestCase):

  def testBuiltin(self):
    print(ParseAndExecute('echo hi'))


if __name__ == '__main__':
  unittest.main()

core/main_loop.py

@@ -3,26 +3,22 @@
"""
main_loop.py

Two variants:

main_loop.Interactive()
main_loop.Batch()

They should call 

ex.ExecuteOne() -- and catch FatalRuntimeError or ControlFlow?

For interactive, you keep going on an error, but exit on 'exit'

For batch, you fail at the first error
They call CommandParser.ParseOne() and Executor.ExecuteAndCatch().

Get rid of:

ex.Execute() -- used for ExitTrap
ex.ExecuteAndCatch()
ParseWholeFile -- needs to check the here doc.
ex.Execute() -- only used for tests
ParseWholeFile() -- needs to check the here doc.
"""

from core import ui
from core import util
from osh.meta import ast

log = util.log

@@ -66,7 +62,7 @@ def Interactive(opts, ex, c_parser, arena):
    return status  # could be a parse error


def Batch(opts, ex, c_parser, arena, nodes_out=None):
def Batch(ex, c_parser, arena, nodes_out=None):
  """Loop for batch execution.

  Args:
@@ -117,3 +113,22 @@ def Batch(opts, ex, c_parser, arena, nodes_out=None):
    return ex.LastStatus()
  else:
    return status  # could be a parse error


def ParseWholeFile(c_parser):
  """Parse an entire shell script.

  This uses the same logic as Batch().
  """
  children = []
  while True:
    node = c_parser.ParseOne()  # can raise ParseError
    if node is None:  # EOF
      c_parser.CheckForPendingHereDocs()  # can raise ParseError
      break
    children.append(node)

  if len(children) == 1:
    return children[0]
  else:
    return ast.CommandList(children)

osh/cmd_parse.py

@@ -1510,32 +1510,22 @@ def CheckForPendingHereDocs(self):
      node = self.pending_here_docs[0]  # Just show the first one?
      p_die('Unterminated here doc began here', word=node.here_begin)

  def ParseWholeFile(self):
    """Entry point for main() in non-interactive shell.

    Very similar to ParseCommandList, but we allow empty files.

    TODO: This should be turned into a Parse and Execute loop, freeing arenas
    if they don't contain functions.

    osh -n is a different loop -- it parses each line one at a time, but
    doesn't execute!
  def ParseCommandSub(self):
    """Parse $(echo hi) and `echo hi`.

    They can have multiple lines, like this:
    echo $(
      echo one
      echo two
    )
    """
    self._NewlineOk()

    # An empty node to execute
    if self.c_kind == Kind.Eof:
    if self.c_kind == Kind.Eof:  # e.g. $()
      return ast.NoOp()

    # This calls ParseAndOr(), but I think it should be a loop that calls
    # ParseCommandLine(), like oil.InteractiveLoop.
    node = self.ParseCommandTerm()
    assert node is not None

    # NOTE: This happens when there is no newline at the end of a file, like
    # osh -c 'cat <<EOF'
    if self.pending_here_docs:
      node = self.pending_here_docs[0]  # Just show the first one?
      p_die('Unterminated here doc began here', word=node.here_begin)

    return node

osh/word_parse.py

@@ -48,14 +48,13 @@
  e.g. "${x:-a  "b"}".
"""

from osh.meta import Id, Kind, LookupKind
from core import braces
from core import word
from core import tdop
from core import util

from osh import arith_parse
from osh.meta import ast, types
from osh.meta import ast, types, Id, Kind, LookupKind

word_part_e = ast.word_part_e
word_e = ast.word_e
@@ -666,7 +665,9 @@ def _ReadCommandSubPart(self, token_type):
    from osh import parse_lib
    c_parser = parse_lib.MakeParserForCommandSub(self.line_reader, self.lexer)

    node = c_parser.ParseWholeFile()  # `` and $() allowed
    # NOTE: This doesn't use something like main_loop because we don't want to
    # interleave parsing and execution!  Unlike 'source' and 'eval'.
    node = c_parser.ParseCommandSub()  # `` and $() allowed
    assert node is not None

    # Hm this creates its own word parser, which is thrown away?