[builtins/mapfile] Fixes due to Julia use case

- Fix reading from stdin. - This is a Python issue -- we don't want to use mylib.Stdin(). - Allow : sigil on variable name - Move tests to spec/builtin-bash
oilshell · Jan 8, 2021 · 4e25aba · 4e25aba
1 parent 1c3de58
commit 4e25aba
Show file tree

Hide file tree

Showing 6 changed files with 236 additions and 155 deletions.
diff --git a/core/process_test.py b/core/process_test.py
@@ -81,11 +81,11 @@ def testStdinRedirect(self):
                  redirect_arg.Path(PATH))
 
     fd_state.Push([r], waiter)
-    line1, _ = builtin_misc.ReadLineFromStdin('\n')
+    line1, _ = builtin_misc.ReadUntilDelim('\n')
     fd_state.Pop()
 
     fd_state.Push([r], waiter)
-    line2, _ = builtin_misc.ReadLineFromStdin('\n')
+    line2, _ = builtin_misc.ReadUntilDelim('\n')
     fd_state.Pop()
 
     # sys.stdin.readline() would erroneously return 'two' because of buffering.

diff --git a/osh/builtin_misc.py b/osh/builtin_misc.py
@@ -143,7 +143,7 @@ def _AppendParts(s, spans, max_results, join_next, parts):
 # NOTE that dash, mksh, and zsh all read a single byte at a time.  It appears
 # to be required by POSIX?  Could try libc getline and make this an option.
 
-def ReadLineFromStdin(delim_char):
+def ReadUntilDelim(delim_char):
   # type: (Optional[str]) -> Tuple[str, bool]
   """Read a portion of stdin.
   
@@ -166,6 +166,22 @@ def ReadLineFromStdin(delim_char):
   return ''.join(chars), eof
 
 
+def ReadLineFromStdin():
+  # type: () -> str
+  chars = []  # type: List[str]
+  while True:
+    c = posix.read(0, 1)
+    if len(c) == 0:
+      break
+
+    chars.append(c)
+
+    if c == '\n':
+      break
+
+  return ''.join(chars)
+
+
 def _ReadLine():
   # type: () -> str
   """Read a line from stdin."""
@@ -385,7 +401,7 @@ def _Read(self, arg, names):
     join_next = False
     status = 0
     while True:
-      line, eof = ReadLineFromStdin(delim_char)
+      line, eof = ReadUntilDelim(delim_char)
 
       if eof:
         # status 1 to terminate loop.  (This is true even though we set
@@ -426,7 +442,6 @@ def __init__(self, mem, errfmt):
     # type: (Mem, ErrorFormatter) -> None
     self.mem = mem
     self.errfmt = errfmt
-    self.f = mylib.Stdin()
 
   def Run(self, cmd_val):
     # type: (cmd_value__Argv) -> int
@@ -436,10 +451,13 @@ def Run(self, cmd_val):
     var_name, _ = arg_r.Peek2()
     if var_name is None:
       var_name = 'MAPFILE'
+    else:
+     if var_name.startswith(':'):
+       var_name = var_name[1:]
 
     lines = []  # type: List[str]
     while True:
-      line = self.f.readline()
+      line = ReadLineFromStdin()
       if len(line) == 0:
         break
       if arg.t and line.endswith('\n'):

diff --git a/spec/builtin-bash.test.sh b/spec/builtin-bash.test.sh
@@ -175,3 +175,161 @@ type -f mv tar grep
 mv is /tmp/mv
 tar is /tmp/tar
 grep is /tmp/grep
+## END
+
+#### mapfile
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s\n' {1..5..2} | {
+  mapfile
+  echo "n=${#MAPFILE[@]}"
+  printf '[%s]\n' "${MAPFILE[@]}"
+}
+## STDOUT:
+n=3
+[1
+]
+[3
+]
+[5
+]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### readarray (synonym for mapfile)
+type readarray >/dev/null 2>&1 || exit 0
+printf '%s\n' {1..5..2} | {
+  readarray
+  echo "n=${#MAPFILE[@]}"
+  printf '[%s]\n' "${MAPFILE[@]}"
+}
+## STDOUT:
+n=3
+[1
+]
+[3
+]
+[5
+]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile (array name): arr
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s\n' {1..5..2} | {
+  mapfile arr
+  echo "n=${#arr[@]}"
+  printf '[%s]\n' "${arr[@]}"
+}
+## STDOUT:
+n=3
+[1
+]
+[3
+]
+[5
+]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile (delimiter): -d delim
+# Note: Bash-4.4+
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s:' {1..5..2} | {
+  mapfile -d : arr
+  echo "n=${#arr[@]}"
+  printf '[%s]\n' "${arr[@]}"
+}
+## STDOUT:
+n=3
+[1:]
+[3:]
+[5:]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile (delimiter): -d '' (null-separated)
+# Note: Bash-4.4+
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s\0' {1..5..2} | {
+  mapfile -d '' arr
+  echo "n=${#arr[@]}"
+  printf '[%s]\n' "${arr[@]}"
+}
+## STDOUT:
+n=3
+[1]
+[3]
+[5]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile (truncate delim): -t
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s\n' {1..5..2} | {
+  mapfile -t arr
+  echo "n=${#arr[@]}"
+  printf '[%s]\n' "${arr[@]}"
+}
+## STDOUT:
+n=3
+[1]
+[3]
+[5]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile -t doesn't remove \r
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s\r\n' {1..5..2} | {
+  mapfile -t arr
+  argv.py "${arr[@]}"
+}
+## STDOUT:
+['1\r', '3\r', '5\r']
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile (store position): -O start
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s\n' a{0..2} | {
+  arr=(x y z)
+  mapfile -O 2 -t arr
+  echo "n=${#arr[@]}"
+  printf '[%s]\n' "${arr[@]}"
+}
+## STDOUT:
+n=5
+[x]
+[y]
+[a0]
+[a1]
+[a2]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile (input range): -s start -n count
+type mapfile >/dev/null 2>&1 || exit 0
+printf '%s\n' a{0..10} | {
+  mapfile -s 5 -n 3 -t arr
+  echo "n=${#arr[@]}"
+  printf '[%s]\n' "${arr[@]}"
+}
+## STDOUT:
+n=3
+[a5]
+[a6]
+[a7]
+## END
+## N-I dash/mksh/zsh/ash stdout-json: ""
+
+#### mapfile / readarray stdin  TODO: Fix me.
+shopt -s lastpipe  # for bash
+
+seq 2 | mapfile m
+seq 3 | readarray r
+echo ${#m[@]}
+echo ${#r[@]}
+## STDOUT:
+2
+3
+## END
diff --git a/spec/builtin-io.test.sh b/spec/builtin-io.test.sh
@@ -696,153 +696,6 @@ var=h
 var=
 ## END
 
-#### mapfile
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s\n' {1..5..2} | {
-  mapfile
-  echo "n=${#MAPFILE[@]}"
-  printf '[%s]\n' "${MAPFILE[@]}"
-}
-## STDOUT:
-n=3
-[1
-]
-[3
-]
-[5
-]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### readarray (synonym for mapfile)
-type readarray >/dev/null 2>&1 || exit 0
-printf '%s\n' {1..5..2} | {
-  readarray
-  echo "n=${#MAPFILE[@]}"
-  printf '[%s]\n' "${MAPFILE[@]}"
-}
-## STDOUT:
-n=3
-[1
-]
-[3
-]
-[5
-]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### mapfile (array name): arr
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s\n' {1..5..2} | {
-  mapfile arr
-  echo "n=${#arr[@]}"
-  printf '[%s]\n' "${arr[@]}"
-}
-## STDOUT:
-n=3
-[1
-]
-[3
-]
-[5
-]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### mapfile (delimiter): -d delim
-# Note: Bash-4.4+
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s:' {1..5..2} | {
-  mapfile -d : arr
-  echo "n=${#arr[@]}"
-  printf '[%s]\n' "${arr[@]}"
-}
-## STDOUT:
-n=3
-[1:]
-[3:]
-[5:]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### mapfile (delimiter): -d '' (null-separated)
-# Note: Bash-4.4+
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s\0' {1..5..2} | {
-  mapfile -d '' arr
-  echo "n=${#arr[@]}"
-  printf '[%s]\n' "${arr[@]}"
-}
-## STDOUT:
-n=3
-[1]
-[3]
-[5]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### mapfile (truncate delim): -t
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s\n' {1..5..2} | {
-  mapfile -t arr
-  echo "n=${#arr[@]}"
-  printf '[%s]\n' "${arr[@]}"
-}
-## STDOUT:
-n=3
-[1]
-[3]
-[5]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### mapfile -t doesn't remove \r
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s\r\n' {1..5..2} | {
-  mapfile -t arr
-  argv.py "${arr[@]}"
-}
-## STDOUT:
-['1\r', '3\r', '5\r']
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### mapfile (store position): -O start
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s\n' a{0..2} | {
-  arr=(x y z)
-  mapfile -O 2 -t arr
-  echo "n=${#arr[@]}"
-  printf '[%s]\n' "${arr[@]}"
-}
-## STDOUT:
-n=5
-[x]
-[y]
-[a0]
-[a1]
-[a2]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-#### mapfile (input range): -s start -n count
-type mapfile >/dev/null 2>&1 || exit 0
-printf '%s\n' a{0..10} | {
-  mapfile -s 5 -n 3 -t arr
-  echo "n=${#arr[@]}"
-  printf '[%s]\n' "${arr[@]}"
-}
-## STDOUT:
-n=3
-[a5]
-[a6]
-[a7]
-## END
-## N-I dash/mksh/zsh/ash stdout-json: ""
-
-
-
 #### read -r -d '' for NUL strings, e.g. find -print0