Open main .sh and libs outside the reserved descriptor range.

The shell should only use descriptors greater than 10. Users can use descriptors 3-9. Also: - Switch here-doc.test.sh to the new format.
oilshell · Jan 13, 2018 · fb9db5c6f346d7113a7818363e47cfcd20441af6 · fb9db5c
1 parent 09a8e1a
commit fb9db5c6f346d7113a7818363e47cfcd20441af6
Unified Split

Showing with 124 additions and 36 deletions.

+6 −7 bin/oil.py

+17 −12 core/cmd_exec.py

+3 −2 core/cmd_exec_test.py

+19 −0 core/process.py

+5 −0 gold/fd-lib.sh

+4 −0 gold/fd-main.sh

+70 −15 spec/here-doc.test.sh
diff --git a/bin/oil.py b/bin/oil.py
@@ -73,6 +73,7 @@ def _tlog(msg):
 from core.id_kind import Id
 from core import legacy
 from core import lexer  # for tracing
+from core import process
 from core import reader
 from core import state
 from core import word
@@ -256,8 +257,9 @@ def OshMain(argv, login_shell):
  # TODO: How to get a handle to initialized builtins here?
  # tokens.py has it.  I think you just make a separate table, with
  # metaprogramming.
-  ex = cmd_exec.Executor(
-      mem, status_lines, funcs, completion, comp_lookup, exec_opts, arena)
+  fd_state = process.FdState()
+  ex = cmd_exec.Executor(mem, fd_state, status_lines, funcs, completion,
+                         comp_lookup, exec_opts, arena)

  # NOTE: The rc file can contain both commands and functions... ideally we
  # would only want to save nodes/lines for the functions.
@@ -307,12 +309,9 @@ def OshMain(argv, login_shell):
        interactive = False
    else:
      arena.PushSource(script_name)
-      # TODO: Does this open file descriptor need to be moved beyond 3..9 ?
-      # Yes!  See dash input.c setinputfile.  It calls savefd().
-      # TODO: It also needs to be closed later.
      try:
-        f = open(script_name)
-      except IOError as e:
+        f = fd_state.Open(script_name)
+      except OSError as e:
        util.error("Couldn't open %r: %s", script_name, os.strerror(e.errno))
        return 1
      line_reader = reader.FileLineReader(f, arena)
diff --git a/core/cmd_exec.py b/core/cmd_exec.py
@@ -102,11 +102,12 @@ class Executor(object):
  It also does some double-dispatch by passing itself into Eval() for
  CompoundWord/WordPart.
  """
-  def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
-               exec_opts, arena):
+  def __init__(self, mem, fd_state, status_lines, funcs, completion,
+               comp_lookup, exec_opts, arena):
    """
    Args:
      mem: Mem instance for storing variables
+      fd_state: FdState() for managing descriptors
      status_lines: shared with completion.  TODO: Move this to the end.
      funcs: registry of functions (these names are completed)
      completion: completion module, if available
@@ -115,6 +116,7 @@ def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
      arena: for printing error locations
    """
    self.mem = mem
+    self.fd_state = fd_state
    self.status_lines = status_lines  
    # function space is different than var space.  Not hierarchical.
    self.funcs = funcs
@@ -132,7 +134,6 @@ def __init__(self, mem, status_lines, funcs, completion, comp_lookup,
    self.bool_ev = expr_eval.BoolEvaluator(mem, exec_opts, self.word_ev)

    self.traps = {}  # signal/hook -> LST node
-    self.fd_state = process.FdState()
    self.dir_stack = state.DirStack()

    # TODO: Pass these in from main()
@@ -213,20 +214,24 @@ def _Source(self, argv):
      return 1

    try:
-      with open(path) as f:
-        line_reader = reader.FileLineReader(f, self.arena)
-        _, c_parser = parse_lib.MakeParser(line_reader, self.arena)
-        # TODO: We have to set a bit to know that we are in 'source'
-        return self._EvalHelper(c_parser, path)
+      f = self.fd_state.Open(path)  # Shell can't use descriptors 3-9
+    except OSError as e:
+      # TODO: Should point to the source statement that failed.
+      util.error('source %r failed: %s', path, os.strerror(e.errno))
+      return 1
+
+    try:
+      line_reader = reader.FileLineReader(f, self.arena)
+      _, c_parser = parse_lib.MakeParser(line_reader, self.arena)
+      return self._EvalHelper(c_parser, path)
+
    except _ControlFlow as e:
      if e.IsReturn():
        return e.ReturnValue()
      else:
        raise
-    except IOError as e:
-      # TODO: Should point to the source statement that failed.
-      util.error('source %r failed: %s', path, os.strerror(e.errno))
-      return 1
+    finally:
+      f.close()

  def _Exec(self, argv):
    # Either execute command with redirects, or apply redirects in this shell.
diff --git a/core/cmd_exec_test.py b/core/cmd_exec_test.py
@@ -44,13 +44,14 @@ def InitExecutor(arena=None):
    arena = test_lib.MakeArena('<InitExecutor>')

  mem = state.Mem('', [], {}, None)
+  fd_state = process.FdState()
  status_lines = None  # not needed for what we're testing
  builtins = builtin.BUILTIN_DEF
  funcs = {}
  comp_funcs = {}
  exec_opts = state.ExecOpts(mem)
-  return cmd_exec.Executor(mem, status_lines, funcs, completion, comp_funcs,
-                           exec_opts, arena)
+  return cmd_exec.Executor(mem, fd_state, status_lines, funcs, completion,
+                           comp_funcs, exec_opts, arena)


 def InitEvaluator():
diff --git a/core/process.py b/core/process.py
@@ -51,6 +51,22 @@ def __init__(self, next_fd=10):
    self.cur_frame = _FdFrame()  # for the top level
    self.stack = [self.cur_frame]

+  def Open(self, path):
+    """Opens a path for read, but moves it out of the reserved 3-9 fd range.
+
+    Returns:
+      A Python file object.  The caller is responsible for Close().
+
+    Raises:
+      OSError if the path can't be found.
+    """
+    fd = os.open(path, os.O_RDONLY, 0666)
+    new_fd = self.next_fd
+    os.dup2(fd, new_fd)
+    self.next_fd += 1
+    os.close(fd)
+    return os.fdopen(new_fd)
+
  def _PushDup(self, fd1, fd2):
    """
    Save fd2 and dup fd1 onto fd2.
@@ -147,6 +163,9 @@ def _ApplyRedirect(self, r, waiter):

      if not self._PushDup(read_fd, r.fd):  # stdin is now the pipe
        ok = False
+
+      # We can't close liek we do in the filename case above?  The writer can
+      # get a "broken pipe".
      self._PushClose(read_fd)

      thunk = _HereDocWriterThunk(write_fd, r.body)
diff --git a/gold/fd-lib.sh b/gold/fd-lib.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. gold/open-fds.sh
+count_func
+
diff --git a/gold/fd-main.sh b/gold/fd-main.sh
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+. gold/fd-lib.sh
+
diff --git a/spec/here-doc.test.sh b/spec/here-doc.test.sh
@@ -33,7 +33,10 @@ fd0
 EOF
 fd3
 EOF3
-# stdout-json: "0: fd0\n3: fd3\n"
+## STDOUT:
+0: fd0
+3: fd3
+## END

 ### Here doc with bad var delimiter
 cat <<${a}
@@ -72,7 +75,11 @@ var: ${var}
 command: $(echo hi)
 arith: $((1+2))
 EOF
-# stdout-json: "var: v\ncommand: hi\narith: 3\n"
+## STDOUT:
+var: v
+command: hi
+arith: 3
+## END

 ### Here doc in middle.  And redirects in the middle.
 # This isn't specified by the POSIX grammar, but it's accepted by both dash and
@@ -82,14 +89,21 @@ echo bar > _tmp/bar.txt
 cat <<EOF 1>&2 _tmp/foo.txt - _tmp/bar.txt
 here
 EOF
-# stderr-json: "foo\nhere\nbar\n"
+## STDERR:
+foo
+here
+bar
+## END

 ### Here doc line continuation
 cat <<EOF \
 ; echo two
 one
 EOF
-# stdout-json: "one\ntwo\n"
+## STDOUT:
+one
+two
+## END

 ### Here doc with quote expansion in terminator
 cat <<'EOF'"2"
@@ -103,7 +117,11 @@ cat <<EOF; echo "two
 three"
 one
 EOF
-# stdout-json: "one\ntwo\nthree\n"
+## STDOUT:
+one
+two
+three
+## END

 ### Two here docs -- first is ignored; second ones wins!
 <<EOF1 cat <<EOF2
@@ -129,7 +147,11 @@ cat <<EOF | tac
 2
 3
 EOF
-# stdout-json: "3\n2\n1\n"
+## STDOUT:
+3
+2
+1
+## END

 ### Here doc with pipe continued on last line
 cat <<EOF |
@@ -138,7 +160,11 @@ cat <<EOF |
 3
 EOF
 tac
-# stdout-json: "3\n2\n1\n"
+## STDOUT:
+3
+2
+1
+## END

 ### Here doc with builtin 'read'
 # read can't be run in a subshell.
@@ -156,7 +182,11 @@ done <<EOF
 2
 3
 EOF
-# stdout-json: "X 1\nX 2\nX 3\n"
+## STDOUT:
+X 1
+X 2
+X 3
+## END


 ### Here doc in while condition and here doc in body
@@ -167,7 +197,11 @@ E1
 E2
 3
 E3
-# stdout-json: "1\n2\n3\n"
+## STDOUT:
+1
+2
+3
+## END

 ### Here doc in while condition and here doc in body on multiple lines
 while cat <<E1 && cat <<E2
@@ -181,7 +215,11 @@ do
 E3
  break
 done
-# stdout-json: "1\n2\n3\n"
+## STDOUT:
+1
+2
+3
+## END

 ### Here doc in while loop split up more
 while cat <<E1
@@ -198,7 +236,11 @@ do
 E3
  break
 done
-# stdout-json: "1\n2\n3\n"
+## STDOUT:
+1
+2
+3
+## END

 ### Mixing << and <<-
 cat <<-EOF; echo --; cat <<EOF2
@@ -252,7 +294,11 @@ cat <<-EOF
 	2
  3
 EOF
-# stdout-json: "1\n2\n  3\n"
+## STDOUT:
+1
+2
+  3
+## END

 ### Here doc within subshell with boolean
 [[ $(cat <<EOF
@@ -268,7 +314,10 @@ here doc in IF CONDITION
 EOF
  echo THEN executed
 fi
-# stdout-json: "here doc in IF CONDITION\nTHEN executed\n"
+## STDOUT:
+here doc in IF CONDITION
+THEN executed
+## END

 ### Multiple here docs in pipeline
 # SKIPPED: hangs with osh on Debian
@@ -278,7 +327,10 @@ fd3
 EOF3
 fd5
 EOF5
-# stdout-json: "0: 3: fd3\n5: fd5\n"
+## STDOUT:
+0: 3: fd3
+5: fd5
+## END

 ### Multiple here docs in pipeline on multiple lines
 # SKIPPED: hangs with osh on Debian
@@ -289,4 +341,7 @@ EOF3
 read_from_fd.py 0 5 5<<EOF5
 fd5
 EOF5
-# stdout-json: "0: 3: fd3\n5: fd5\n"
+## STDOUT:
+0: 3: fd3
+5: fd5
+## END