Fix 'source' and 'eval' syntax errors to give location info.

Also, error locations work inside here doc!

Demos in test/runtime-errors.sh for these cases.  TODO: Turn these into
proper tests.

Details:

- New, cleaner parse_lib interface.
- New FileLineReader abstraction.
  - Express StringLineReader in terms of FileLineReader
  - Thoroughly test the line readers and fix bugs!
- New arena.PushSource / PopSource API

Other:

- Clarify the usage of the readline interface, as it relates to the OSH
  parser.
- Catch exception for 'osh nonexistent.sh'

Tests:

- Add test cases for 'source' and 'eval' syntax errors.
  - Document what different shells do.  bash does the best.
- Move 'eval' test to builtins.test.sh.
- Add unit tests for alloc and reader.

Unit tests and spec tests all pass.

bin/oil.py

@@ -239,27 +239,26 @@ def OshMain(argv):
  # tokens.py has it.  I think you just make a separate table, with
  # metaprogramming.
  ex = cmd_exec.Executor(
      mem, status_lines, funcs, completion, comp_lookup, exec_opts,
      parse_lib.MakeParserForExecutor, arena)
      mem, status_lines, funcs, completion, comp_lookup, exec_opts, arena)

  # NOTE: The rc file can contain both commands and functions... ideally we
  # would only want to save nodes/lines for the functions.
  try:
    rc_path = 'oilrc'
    arena.PushSource(rc_path)
    with open(rc_path) as f:
      contents = f.read()
    arena.AddSourcePath(rc_path)
    #print(repr(contents))

    rc_line_reader = reader.StringLineReader(contents, arena=arena)
    _, rc_c_parser = parse_lib.MakeParserForTop(rc_line_reader)
    rc_node = rc_c_parser.ParseWholeFile()
    if not rc_node:
      # TODO: Error should return a token, and then the token should have a
      # arena index, and then look that up in the arena.
      err = rc_c_parser.Error()
      ui.PrintErrorStack(err, arena, sys.stderr)
      return 2  # parse error is code 2
      rc_line_reader = reader.FileLineReader(f, arena=arena)
      _, rc_c_parser = parse_lib.MakeParser(rc_line_reader, arena)
      try:
        rc_node = rc_c_parser.ParseWholeFile()
        if not rc_node:
          # TODO: Error should return a token, and then the token should have a
          # arena index, and then look that up in the arena.
          err = rc_c_parser.Error()
          ui.PrintErrorStack(err, arena, sys.stderr)
          return 2  # parse error is code 2
      finally:
        arena.PopSource()

    status = ex.Execute(rc_node)
    #print('oilrc:', status, cflow, file=sys.stderr)
@@ -269,34 +268,41 @@ def OshMain(argv):
      raise

  if opts.c is not None:
    arena.AddSourcePath('<command string>')
    arena.PushSource('<command string>')
    line_reader = reader.StringLineReader(opts.c, arena=arena)
    interactive = False
  elif opts.i:  # force interactive
    arena.AddSourcePath('<stdin -i>')
    arena.PushSource('<stdin -i>')
    line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena)
    interactive = True
  else:
    try:
      script_name = argv[opt_index]
    except IndexError:
      if sys.stdin.isatty():
        arena.AddSourcePath('<interactive>')
        arena.PushSource('<interactive>')
        line_reader = reader.InteractiveLineReader(OSH_PS1, arena=arena)
        interactive = True
      else:
        arena.AddSourcePath('<stdin>')
        line_reader = reader.StringLineReader(sys.stdin.read(), arena=arena)
        arena.PushSource('<stdin>')
        line_reader = reader.FileLineReader(sys.stdin, arena=arena)
        interactive = False
    else:
      arena.AddSourcePath(script_name)
      with open(script_name) as f:
        line_reader = reader.StringLineReader(f.read(), arena=arena)
      arena.PushSource(script_name)
      # TODO: Does this open file descriptor need to be moved beyond 3..9 ?
      # Yes!  See dash input.c setinputfile.  It calls savefd().
      # TODO: It also needs to be closed later.
      try:
        f = open(script_name)
      except IOError as e:
        util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno))
        return 1
      line_reader = reader.FileLineReader(f, arena=arena)
      interactive = False

  # TODO: assert arena.NumSourcePaths() == 1
  # TODO: .rc file needs its own arena.
  w_parser, c_parser = parse_lib.MakeParserForTop(line_reader, arena=arena)
  w_parser, c_parser = parse_lib.MakeParser(line_reader, arena)

  if interactive:
    # NOTE: We're using a different evaluator here.  The completion system can

core/alloc.py

@@ -35,30 +35,40 @@ def __init__(self, arena_id):
    self.spans = []
    self.next_span_id = 0

    self.debug_info = []  # list of (src_path index, physical line number)
    # List of (src_path index, physical line number).  This is two integers for
    # every line read.  We could use a clever encoding of this.  (Although the
    # it's probably more important to compact the ASDL representation.)
    self.debug_info = []
    self.src_paths = []  # list of source paths
    self.src_index = -1  # index of current source file
    self.src_id_stack = []  # stack of src_id integers

  def IsComplete(self):
    """Return whether we have a full set of lines -- none of which was cleared.

    Maybe just an assertion error.
    """

  def AddSourcePath(self, src_path):
    # TODO: Should this be part of the pool?
  def PushSource(self, src_path):
    src_id = len(self.src_paths)
    self.src_paths.append(src_path)
    self.src_index += 1
    self.src_id_stack.append(src_id)

  def PopSource(self):
    self.src_id_stack.pop()

  def AddLine(self, line, line_num):
    """
    Args:
      line: string
      line_num: physical line number, for printing

    TODO: Add an option of whether to save the line?  You can retrieve it on
    disk in many cases.
    disk in many cases.  (But not in the stdin, '-c', 'eval' case)
    """
    line_id = self.next_line_id
    self.lines.append(line)
    self.next_line_id += 1
    self.debug_info.append((self.src_index, line_num))
    self.debug_info.append((self.src_id_stack[-1], line_num))
    return line_id

  def ClearLastLine(self):
@@ -90,11 +100,11 @@ def GetLineSpan(self, span_id):
  def GetDebugInfo(self, line_id):
    """Get the path and physical line number, for parse errors."""
    assert line_id >= 0
    src_index, line_num = self.debug_info[line_id]
    src_id , line_num = self.debug_info[line_id]
    try:
      path = self.src_paths[src_index]
      path = self.src_paths[src_id]
    except IndexError:
      print('INDEX', src_index)
      print('INDEX', src_id)
      raise
    return path, line_num

core/alloc_test.py

@@ -0,0 +1,55 @@
#!/usr/bin/env python
"""
alloc_test.py: Tests for alloc.py
"""

import unittest

import alloc  # module under test


class AllocTest(unittest.TestCase):

  def setUp(self):
    p = alloc.Pool()
    self.arena = p.NewArena()

  def testPool(self):
    arena = self.arena
    arena.PushSource('one.oil')

    line_id = arena.AddLine('line 1', 1)
    self.assertEqual(0, line_id)
    line_id = arena.AddLine('line 2', 2)
    self.assertEqual(1, line_id)

    span_id = arena.AddLineSpan(None)
    self.assertEqual(0, span_id)

    arena.PopSource()

    self.assertEqual(('one.oil', 1), arena.GetDebugInfo(0))
    self.assertEqual(('one.oil', 2), arena.GetDebugInfo(1))

  def testPushSource(self):
    arena = self.arena

    arena.PushSource('one.oil')
    arena.AddLine('echo 1a', 1)
    arena.AddLine('source two.oil', 2)

    arena.PushSource('two.oil')
    arena.AddLine('echo 2a', 1)
    id2 = arena.AddLine('echo 2b', 2)  # line 2 of two.oil
    arena.PopSource()

    id3 = arena.AddLine('echo 1c', 3)  # line 3 of one.oil
    arena.PopSource()

    # TODO: fix these assertions
    self.assertEqual(('two.oil', 2), arena.GetDebugInfo(id2))
    self.assertEqual(('one.oil', 3), arena.GetDebugInfo(id3))


if __name__ == '__main__':
  unittest.main()

core/cmd_exec.py

@@ -24,6 +24,7 @@
from core import args
from core import braces
from core import expr_eval
from core import reader
from core import word_eval
from core import ui
from core import util
@@ -35,6 +36,7 @@
from core import state

from osh import ast_ as ast
from osh import parse_lib

import libc  # for fnmatch

@@ -87,15 +89,15 @@ class Executor(object):
  CompoundWord/WordPart.
  """
  def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
               exec_opts, make_parser, arena):
               exec_opts, arena):
    """
    Args:
      mem: Mem instance for storing variables
      status_lines: shared with completion.  TODO: Move this to the end.
      funcs: registry of functions (these names are completed)
      completion: completion module, if available
      comp_lookup: completion pattern/action
      make_parser: Callback for creating a new command parser (eval and source)
      exec_opts: ExecOpts
      arena: for printing error locations
    """
    self.mem = mem
@@ -107,7 +109,6 @@ def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
    self.comp_lookup = comp_lookup
    # This is for shopt and set -o.  They are initialized by flags.
    self.exec_opts = exec_opts
    self.make_parser = make_parser
    self.arena = arena

    self.ev = word_eval.NormalWordEvaluator(mem, exec_opts, self)
@@ -155,25 +156,31 @@ def _Complete(self, argv):
  def _CompGen(self, argv):
    raise NotImplementedError

  def _EvalHelper(self, code_str):
    c_parser = self.make_parser(code_str)
    node = c_parser.ParseWholeFile()
    # NOTE: We could model a parse error as an exception, like Python, so we
    # get a traceback.  (This won't be applicable for a static module system.)
    if not node:
      print('Error parsing code %r' % code_str)
      return 1
    status = self._Execute(node)
    return status
  def _EvalHelper(self, c_parser, source_name):
    self.arena.PushSource(source_name)
    try:
      node = c_parser.ParseWholeFile()
      # NOTE: We could model a parse error as an exception, like Python, so we
      # get a traceback.  (This won't be applicable for a static module system.)
      if not node:
        util.error('Parse error in %r:', source_name)
        err = c_parser.Error()
        ui.PrintErrorStack(err, self.arena, sys.stderr)
        return 1
      status = self._Execute(node)
      return status
    finally:
      self.arena.PopSource()

  def _Eval(self, argv):
    # TODO: in oil, eval shouldn't take multiple args.  For clarity 'eval ls
    # foo' will say "extra arg".
    # NOTE: in oil, eval shouldn't take multiple args.  For clarity, 'eval ls
    # foo' will be an "extra arg" error.
    code_str = ' '.join(argv)
    return self._EvalHelper(code_str)
    line_reader = reader.StringLineReader(code_str, self.arena)
    _, c_parser = parse_lib.MakeParser(line_reader, self.arena)
    return self._EvalHelper(c_parser, '<eval string>')

  def _Source(self, argv):
    # TODO: Use LineReader
    try:
      path = argv[0]
    except IndexError:
@@ -182,12 +189,13 @@ def _Source(self, argv):
      return 1
    try:
      with open(path) as f:
        code_str = f.read()
        line_reader = reader.FileLineReader(f, self.arena)
        _, c_parser = parse_lib.MakeParser(line_reader, self.arena)
        return self._EvalHelper(c_parser, path)
    except IOError as e:
      # TODO: Should point to the source statement that failed.
      util.error('source %r failed: %s', path, os.strerror(e.errno))
      return 1
    return self._EvalHelper(code_str)

  def _Exec(self, argv):
    # Either execute command with redirects, or apply redirects in this shell.

core/cmd_exec_test.py

@@ -45,8 +45,8 @@ def InitExecutor():
  exec_opts = state.ExecOpts()
  pool = alloc.Pool()
  arena = pool.NewArena()
  return cmd_exec.Executor(mem, status_lines, funcs, completion, comp_funcs, exec_opts,
                           parse_lib.MakeParserForExecutor, arena)
  return cmd_exec.Executor(mem, status_lines, funcs, completion, comp_funcs,
                           exec_opts, arena)


def InitEvaluator():

core/completion.py

@@ -653,14 +653,16 @@ def Matches(self, buf, status_out):
class ReadlineCompleter(object):
  def __init__(self, root_comp, status_out, debug=False):
    self.root_comp = root_comp
    self.status_out  = status_out 
    self.status_out = status_out 
    self.debug = debug

    self.comp_iter = None  # current completion being processed

  def _GetNextCompletion(self, word, state):
  def _GetNextCompletion(self, state):
    if state == 0:
      # TODO: Tokenize it according to our language
      # TODO: Tokenize it according to our language.  If this is $PS2, we also
      # need previous lines!  Could make a VirtualLineReader instead of
      # StringLineReader?
      buf = readline.get_line_buffer()

      # Begin: the index of the first char of the 'word' in the line.  Words
@@ -688,12 +690,14 @@ def _GetNextCompletion(self, word, state):

    return next_completion

  def __call__(self, word, state):
  def __call__(self, unused_word, state):
    """Return a single match."""
    # The readline library swallows exceptions.  We have to display them
    # ourselves.
    # NOTE: The readline library tokenizes words.  We bypass that and use
    # get_line_buffer().  So we get 'for x in l' instead of just 'l'.

    #self.status_out.Write(0, 'word %r state %s', unused_word, state)
    try:
      return self._GetNextCompletion(word, state)
      return self._GetNextCompletion(state)
    except Exception as e:
      traceback.print_exc()
      self.status_out.Write(0, 'Unhandled exception while completing: %s', e)

core/lexer_test.py

@@ -36,6 +36,11 @@ def testPrintStats(self):
    print("Number of lex states: %d" % len(LEXER_DEF))
    print("Number of token dispatches: %d" % total)

  def testLineId(self):
    # TODO: Test that the lexer gives line_ids when passed an arena.
    # This might be more relevant if we start deallocating memroy.
    pass


if __name__ == '__main__':
  unittest.main()