Fix bug where we would overwrite existing file descriptors with dup2().

Now we check file descriptors (even those above #10) before using them.

This is a recurrence of the bug described here:

http://www.oilshell.org/blog/2017/07/02.html

If you do 'import random' in Python, it results in /dev/urandom being
permanently open (which can be seen in the blog post!)

At the time, I incorrectly fixed the bug by just opening our own
descriptors above #10.  But it's possible that CPython will subsequently
open something above #10 (/dev/urandom appeared as #12), so we have to
additionally check descriptors before we use them.

NOTE: 'import cgi' eventually causes 'import random'.  We're only using
it for cgi.escape(), but cgi.FieldStorage() uses tempfile, which uses a
random number generator.

Also: minimize the test case as gold/configure-bug.sh.

'test/gold.sh configure' was the case that caught this bug.  The spec
tests didn't find it!  Because it appears to take about 4 redirects for
this to happen!

core/process.py

@@ -51,6 +51,18 @@ def __init__(self, next_fd=10):
    self.cur_frame = _FdFrame()  # for the top level
    self.stack = [self.cur_frame]

  def _NextFreeFileDescriptor(self):
    done = False
    while not done:
      try:
        fcntl.fcntl(self.next_fd, fcntl.F_GETFD)
      except IOError as e:
        if e.errno == errno.EBADF:
          done = True
      self.next_fd += 1

    return self.next_fd

  def Open(self, path):
    """Opens a path for read, but moves it out of the reserved 3-9 fd range.

@@ -61,9 +73,8 @@ def Open(self, path):
      OSError if the path can't be found.
    """
    fd = os.open(path, os.O_RDONLY, 0666)
    new_fd = self.next_fd
    new_fd = self._NextFreeFileDescriptor()
    os.dup2(fd, new_fd)
    self.next_fd += 1
    os.close(fd)
    return os.fdopen(new_fd)

@@ -75,11 +86,12 @@ def _PushDup(self, fd1, fd2):
    Returns:
      success Bool
    """
    new_fd = self._NextFreeFileDescriptor()
    #log('---- _PushDup %s %s', fd1, fd2)
    need_restore = True
    try:
      #log('DUPFD %s %s', fd2, self.next_fd)
      fcntl.fcntl(fd2, fcntl.F_DUPFD, self.next_fd)
      fcntl.fcntl(fd2, fcntl.F_DUPFD, new_fd)
    except IOError as e:
      # Example program that causes this error: exec 4>&1.  Descriptor 4 isn't
      # open.
@@ -91,7 +103,7 @@ def _PushDup(self, fd1, fd2):
        raise
    else:
      os.close(fd2)
      fcntl.fcntl(self.next_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)
      fcntl.fcntl(new_fd, fcntl.F_SETFD, fcntl.FD_CLOEXEC)

    #log('==== dup %s %s\n' % (fd1, fd2))
    try:
@@ -100,14 +112,13 @@ def _PushDup(self, fd1, fd2):
      # bash/dash give this error too, e.g. for 'echo hi 1>&3'
      util.error('%d: %s', fd1, os.strerror(e.errno))
      # Restore and return error
      os.dup2(self.next_fd, fd2)
      os.close(self.next_fd)
      os.dup2(new_fd, fd2)
      os.close(new_fd)
      # Undo it
      return False

    if need_restore:
      self.cur_frame.saved.append((self.next_fd, fd2))
    self.next_fd += 1
      self.cur_frame.saved.append((new_fd, fd2))
    return True

  def _PushClose(self, fd):
@@ -249,6 +260,9 @@ def Pop(self):
        raise
      os.close(saved)
      #log('dup2 %s %s', saved, orig)

      # NOTE: This balances the increments from _PushDup().  But it doesn't
      # balance the ones from Open().
      self.next_fd -= 1  # Count down

    for fd in frame.need_close:

gold/configure-bug.sh

@@ -0,0 +1,13 @@
#!/bin/bash

detect_readline() {
  echo foo >/dev/null 2>&1
  echo "two" 1>&2
}

main() {
  detect_readline > _tmp/f2-out.txt
  #detect_readline 
}

main "$@"

test/gold.sh

@@ -88,6 +88,7 @@ startup-benchmark() {
}

configure() { _compare ./configure; }
configure-bug() { _compare gold/configure-bug.sh; }
nix() { _compare gold/nix.sh isElfSimpleWithStdin; }
and-or() { _compare gold/and-or.sh test-simple; }