Implement and integrate a new algorithm for word splitting.

- It fixes many tests in word-split.test.sh.
- Add unit tests for many corner cases.
- It exposes a fundamental problem with the part_value representation.
  Tests like argv ${x:-"1 2" "3 4"} are now failing.  These will be
  fixed by a new representation.
- It's not yet integrated with the 'read' builtin, but will be shortly.

The new algorithm is an explicit state machine, and the output is an
array of spans.

There is also a simpler WhitespaceSpitter for the common case.  A new
RootSplitter() abstraction manages these splitter instances, as IFS
is scoped.

bin/oil.py

@@ -71,6 +71,7 @@ def _tlog(msg):
from core import builtin
from core import cmd_exec
from core.id_kind import Id
from core import legacy
from core import lexer  # for tracing
from core import reader
from core import state
@@ -325,7 +326,8 @@ def OshMain(argv, login_shell):
    # NOTE: We're using a different evaluator here.  The completion system can
    # also run functions... it gets the Executor through Executor._Complete.
    if HAVE_READLINE:
      ev = word_eval.CompletionWordEvaluator(mem, exec_opts)
      splitter = legacy.CompletionSplitter()
      ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
      status_out = completion.StatusOutput(status_lines, exec_opts)
      completion.Init(pool, builtin.BUILTIN_DEF, mem, funcs, comp_lookup,
                      status_out, ev)

core/builtin.py

@@ -30,6 +30,7 @@
import sys

from core import args
from core import legacy
from core import lexer
from core import runtime
from core import util
@@ -521,21 +522,7 @@ def Jobs(argv, job_state):
READ_SPEC.ShortFlag('-n', args.Int)


def _SplitLine(line, ifs, allow_escape):
  continued = False

  # Or should I just return a list of indices?
  parts = []

  n = len(line)
  for i in xrange(n):
    c = line[i]

  return parts, continued


def Read(argv, mem):

def Read(argv, splitter, mem):
  arg, i = READ_SPEC.Parse(argv)

  if not arg.r:
@@ -570,6 +557,18 @@ def Read(argv, mem):
  # leftover words assigned to the last name
  n = len(names)

  #ifs = legacy.GetIfs(mem)
  parts = splitter.SplitForRead(line, not arg.r)

  # If the last part is ignored and it consists of a single \, then we need to
  # read another line!  And we need to JOIN the other non-ignored segments on
  # adjacent lines!!!

  continued = False
  #log('split: %s %s', parts, continued)
  #

  # TODO: replace this with the above
  strs = line.split(None, n-1)

  # TODO: Use REPLY variable here too?

core/cmd_exec.py

@@ -27,6 +27,7 @@
from core import args
from core import braces
from core import expr_eval
from core import legacy
from core import reader
from core import test_builtin
from core import word
@@ -124,7 +125,9 @@ def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
    self.exec_opts = exec_opts
    self.arena = arena

    self.word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, self)
    self.splitter = legacy.RootSplitter(self.mem)
    self.word_ev = word_eval.NormalWordEvaluator(
        mem, exec_opts, self.splitter, self)
    self.arith_ev = expr_eval.ArithEvaluator(mem, exec_opts, self.word_ev)
    self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.word_ev)

@@ -248,7 +251,7 @@ def _RunBuiltin(self, builtin_id, argv):
      self.fd_state.MakePermanent()

    elif builtin_id == EBuiltin.READ:
      status = builtin.Read(argv, self.mem)
      status = builtin.Read(argv, self.splitter, self.mem)

    elif builtin_id == EBuiltin.ECHO:
      status = builtin.Echo(argv)

core/cmd_exec_test.py

@@ -18,6 +18,7 @@
from core.cmd_exec import *
from core.id_kind import Id
from core import completion
from core import legacy
from core import ui
from core import word_eval
from core import runtime
@@ -59,7 +60,8 @@ def InitEvaluator():

  exec_opts = state.ExecOpts(mem)
  # Don't need side effects for most things
  return word_eval.CompletionWordEvaluator(mem, exec_opts)
  splitter = legacy.CompletionSplitter()
  return word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)


class ExpansionTest(unittest.TestCase):

core/completion_test.py

@@ -15,6 +15,7 @@
from core import cmd_exec
from core import cmd_exec_test
from core import completion  # module under test
from core import legacy
from core import lexer
from core import state
from core import test_lib
@@ -146,7 +147,8 @@ def testRootCompleter(self):
def _MakeTestEvaluator():
  mem = state.Mem('', [], {}, None)
  exec_opts = state.ExecOpts(mem)
  ev = word_eval.CompletionWordEvaluator(mem, exec_opts)
  splitter = legacy.CompletionSplitter()
  ev = word_eval.CompletionWordEvaluator(mem, exec_opts, splitter)
  return ev