Initial implementation of process substitution -- <() and >().

The spec tests now pass, and Aboriginal gets further.  But there's now
another error in the code inside the process substitution that I haven't
debugged yet.

I still have to figure out waiting on the command -- either setting $!
or waiting by default.  The bug might be related to that.

Addresses issue #58.

core/cmd_exec.py

@@ -1110,6 +1110,100 @@ def RunCommandSub(self, node):

    return ''.join(chunks).rstrip('\n')

  def RunProcessSub(self, node, op_id):
    """Process sub creates a forks a process connected to a pipe.

    The pipe is typically passed to another process via a /dev/fd/$FD path.

    TODO:

    sane-proc-sub:
    - wait for all the words

    Otherwise, set $!  (mem.last_job_id)

    strict-proc-sub:
    - Don't allow it anywhere except SimpleCommand, any redirect, or
    Assignment?  And maybe not even assignment?

    Should you put return codes in @PROCESS_SUB_STATUS?  You need two of them.
    """
    p = self._MakeProcess(node)

    r, w = os.pipe()

    if op_id == Id.Left_ProcSubIn:
      # Example: cat < <(head foo.txt)
      #
      # The head process should write its stdout to a pipe.
      redir = process.StdoutToPipe(r, w)

    elif op_id == Id.Left_ProcSubOut:
      # Example: head foo.txt > >(tac)
      #
      # The tac process should read its stdin from a pipe.
      #
      # NOTE: This appears to hang in bash?  At least when done interactively.
      # It doesn't work at all in osh interactively?
      redir = process.StdinFromPipe(r, w)

    else:
      raise AssertionError

    p.AddStateChange(redir)

    # Fork, letting the child inherit the pipe file descriptors.
    pid = p.Start()

    # TODO: Set $!

    # After forking, close the end of the pipe we're not using.
    if op_id == Id.Left_ProcSubIn:
      os.close(w)
    elif op_id == Id.Left_ProcSubOut:
      os.close(r)
    else:
      raise AssertionError

    #log('I am %d', os.getpid())
    #log('Process sub started %d', pid)
    self.waiter.Register(pid, p.WhenDone)

    # Is /dev Linux-specific?
    if op_id == Id.Left_ProcSubIn:
      return '/dev/fd/%d' % r

    elif op_id == Id.Left_ProcSubOut:
      return '/dev/fd/%d' % w

    else:
      raise AssertionError

    # Generalize?
    #
    # - Make it work first, bare minimum.
    # - Then Make something like Pipeline()?
    #   - you add all the argument processes
    #   - then you add the main processes, with those as args
    #   - then p.Wait()
    #     - get status for all of them?
    #
    # Problem is that you don't see this until word_eval?
    # You can scan a simple command for these though.

    # TODO:
    # - Do we need to somehow register a waiter?  After SimpleCommand,
    #   argv and redirect words need to wait?
    # - what about for loops?  case?  ControlFlow?  temp binding,
    #   assignments, etc. They all have words
    #   - disallow those?
    # I guess you need it at the end of every command sub loop?
    # But you want to detect statically if you need to wait?
    # Maybe just have a dirty flag?  needs_wait
      # - Make a pipe
      # - Start another process connected to the write end of the pipe.
      # - Return [/dev/fd/FD] as the read end of the pipe.

  def RunFunc(self, func_node, argv):
    """Used by completion engine."""
    # These are redirects at DEFINITION SITE.  You can also have redirects at

core/word_eval.py

@@ -401,6 +401,17 @@ def _EvalCommandSub(self, part, quoted):
    """
    raise NotImplementedError

  def _EvalProcessSub(self, part, id_):
    """Abstract since it has a side effect.

    Args:
      part: CommandSubPart

    Returns:
       part_value
    """
    raise NotImplementedError

  def _EvalTildeSub(self, prefix):
    """Evaluates ~ and ~user.

@@ -884,12 +895,15 @@ def _EvalWordPart(self, part, quoted=False):
      return [self._EvalDoubleQuotedPart(part)]

    elif part.tag == word_part_e.CommandSubPart:
      if part.left_token.id not in (Id.Left_CommandSub, Id.Left_Backtick):
        # TODO: If token is Id.Left_ProcSubIn or Id.Left_ProcSubOut, we have to
        # supply something like /dev/fd/63.
        raise NotImplementedError(part.left_token.id)
      id_ = part.left_token.id
      if id_ in (Id.Left_CommandSub, Id.Left_Backtick):
        return [self._EvalCommandSub(part.command_list, quoted)]

      return [self._EvalCommandSub(part.command_list, quoted)]
      elif id_ in (Id.Left_ProcSubIn, Id.Left_ProcSubOut):
        return [self._EvalProcessSub(part.command_list, id_)]

      else:
        raise AssertionError(id_)

    elif part.tag == word_part_e.SimpleVarSub:
      decay_array = False
@@ -1144,6 +1158,11 @@ def _EvalCommandSub(self, node, quoted):
    # https://unix.stackexchange.com/questions/17747/why-does-shell-command-substitution-gobble-up-a-trailing-newline-char
    return runtime.StringPartValue(stdout, not quoted, not quoted)

  def _EvalProcessSub(self, node, id_):
    dev_path = self.ex.RunProcessSub(node, id_)
    # no split or glob
    return runtime.StringPartValue(dev_path, False, False)


class NormalWordEvaluator(_WordEvaluator):

@@ -1171,6 +1190,10 @@ def _EvalCommandSub(self, node, quoted):
    return runtime.StringPartValue(
        '__COMMAND_SUB_NOT_EXECUTED__', not quoted, not quoted)

  def _EvalProcessSub(self, node, id_):
    return runtime.StringPartValue(
        '__PROCESS_SUB_NOT_EXECUTED__', False, False)


class CompletionWordEvaluator(_WordEvaluator):

spec/process-sub.test.sh

@@ -3,8 +3,21 @@
### Process sub input
f=_tmp/process-sub.txt
{ echo 1; echo 2; echo 3; } > $f
comm <(head -n 2 $f) <(tail -n 2 $f)
# stdout-json: "1\n\t\t2\n\t3\n"
cat <(head -n 2 $f) <(tail -n 2 $f)
## STDOUT:
1
2
2
3
## END

### Process sub output
{ echo 1; echo 2; echo 3; } > >(tac)
## STDOUT:
3
2
1
## END

### Non-linear pipeline with >()
stdout_stderr() {
@@ -25,4 +38,9 @@ cat $TMP/out.txt
# PROBLEM -- OUT comes first, and then 'warning: e2', and then 'o2 o1'.  It
# looks like it's because nobody waits for the proc sub.
# http://lists.gnu.org/archive/html/help-bash/2017-06/msg00018.html
# stdout-json: "OUT\nwarning: e2\no2\no1\n"
## STDOUT:
OUT
warning: e2
o2
o1
## END