Upgrade assertions to match bash 4.4.

We upgraded a few commits ago for ${x@P}.  We also want ${x@Q}.

- About 5 bash bugs were fixed!  Three bugs in array.test.sh, and few
  others.
- Changed ${!var} in OSH to be more strict, roughly matching bash.
  (Although it is a fatal error now, not exiting 1.)
- Separate extglob pattern matching tests into extglob-match.test.sh.
  File system globbing with extglob remains in extended-glob.test.sh.
- Unit tests for GNU libc extended glob.  It seems to do what we want,
  but we don't want to be dependent on GNU libc.  musl libc doesn't have
  it.

core/expr_eval.py

@@ -699,12 +699,12 @@ def Eval(self, node):
         raise NotImplementedError(op_id)
 
       if arg_type == bool_arg_type_e.Str:
-        # TODO:
-        # - Compare arrays.  (Although bash coerces them to string first)
 
         if op_id in (Id.BoolBinary_GlobEqual, Id.BoolBinary_GlobDEqual):
           #log('Comparing %s and %s', s2, s1)
-          # TODO: Respect extended glob?
+
+          # TODO: Respect extended glob?  * and ! and ? are quoted improperly.
+          # But @ and + are OK.
           return libc.fnmatch(s2, s1)
 
         if op_id == Id.BoolBinary_GlobNEqual:

core/glob_.py

@@ -15,6 +15,13 @@
 log = util.log
 glob_part_e = ast.glob_part_e
 
+# TODO: Need LooksLikeExtGlob?
+#
+# pat='@(foo|bar)'
+# [[ foo == $pat ]] -- this works.
+#
+# Problem with extended glob -> ERE
+# x!(foo|bar)y
 
 def LooksLikeGlob(s):
   """

core/word_eval.py

@@ -9,12 +9,12 @@
 from core import expr_eval
 from core import libstr
 from core import glob_
-from osh.meta import Id, Kind, LookupKind
-from osh.meta import runtime
 from core import state
 from core import word_compile
 from core import util
-from osh.meta import ast
+
+from osh.meta import Id, Kind, LookupKind, ast, runtime
+from osh import match
 
 word_e = ast.word_e
 bracket_op_e = ast.bracket_op_e
@@ -347,6 +347,11 @@ def _ApplyPrefixOp(self, val, op_id):
           arg_num = int(val.s)
           return self.mem.GetArgNum(arg_num)
         except ValueError:
+          if not match.IsValidVarName(val.s):
+            # TODO: location information.
+            # Also note that bash doesn't consider this fatal.  It makes the
+            # command exit with '1', but we don't have that ability yet?
+            e_die('Bad variable name %r in var ref', val.s)
           return self.mem.GetVar(val.s)
       elif val.tag == value_e.StrArray:
         raise NotImplementedError('${!a[@]}')  # bash gets keys this way

native/libc_test.py

@@ -42,6 +42,35 @@ def testFnmatch(self):
       actual = libc.fnmatch(pat, s)
       self.assertEqual(expected, actual)
 
+  def testFnmatchExtglob(self):
+    return
+
+    # With GNU extension.
+    cases = [
+        # One of these
+        ('--@(help|verbose)', '--verbose', 1),
+        ('--@(help|verbose)', '--foo', 0),
+
+        ('--*(help|verbose)', '--verbose', 1),
+        ('--*(help|verbose)', '--', 1),
+        ('--*(help|verbose)', '--helpverbose', 1),  # Not what we want
+
+        ('--+(help|verbose)', '--verbose', 1),
+        ('--+(help|verbose)', '--', 0),
+        ('--+(help|verbose)', '--helpverbose', 1),  # Not what we want
+
+        ('--?(help|verbose)', '--verbose', 1),
+        ('--?(help|verbose)', '--helpverbose', 0),
+
+        # Neither of these
+        ('--!(help|verbose)', '--verbose', 0),
+    ]
+    for pat, s, expected in cases:
+      actual = libc.fnmatch(pat, s)
+      self.assertEqual(expected, actual,
+          "Matching %s against %s: got %s but expected %s" %
+          (pat, s, actual, expected))
+
   def testGlob(self):
     print(libc.glob('*.py'))
 

spec/array.test.sh

@@ -31,14 +31,17 @@ argv.py ${empty[@]:-not one} "${empty[@]:-not one}"
 
 #### nounset with empty array (design bug, makes it hard to use arrays)
 # http://lists.gnu.org/archive/html/help-bash/2017-09/msg00005.html
-# TODO: sane-arrays should get rid of this problem.
+# NOTE: This used to be a bug in bash 4.3, but is fixed in bash 4.4.
 set -o nounset
 empty=()
 argv.py "${empty[@]}"
 echo status=$?
-## stdout-json: "[]\nstatus=0\n"
-## BUG bash/mksh stdout-json: ""
-## BUG bash/mksh status: 1
+## STDOUT:
+[]
+status=0
+## END
+## BUG mksh stdout-json: ""
+## BUG mksh status: 1
 
 #### local array
 # mksh support local variables, but not local arrays, oddly.
@@ -311,24 +314,20 @@ argv.py "${files[@]%.c}"
 ## N-I mksh stdout-json: ""
 
 #### Multiple subscripts not allowed
+# NOTE: bash 4.3 had a bug where it ignored the bad subscript, but now it is
+# fixed.
 a=('123' '456')
 argv.py "${a[0]}" "${a[0][0]}"
 ## stdout-json: ""
 ## status: 2
-## OK mksh status: 1
-# bash is bad -- it IGNORES the bad subscript.
-## BUG bash status: 0
-## BUG bash stdout: ['123', '123']
+## OK bash/mksh status: 1
 
 #### Length op, index op, then transform op is not allowed
 a=('123' '456')
 echo "${#a[0]}" "${#a[0]/1/xxx}"
 ## stdout-json: ""
 ## status: 2
-## OK mksh status: 1
-# bash is bad -- it IGNORES the op at the end
-## BUG bash status: 0
-## BUG bash stdout: 3 3
+## OK bash/mksh status: 1
 
 #### Array subscript not allowed on string
 s='abc'

spec/assoc.test.sh

@@ -29,7 +29,7 @@ declare -A a
 a=([aa]=b [foo]=bar ['a+1']=c)
 argv.py "${!a[@]}"
 # Is this invalid on associative arrays?  Makes no sense.
-## stdout: ['aa', 'foo', 'a+1']
+## stdout: ['foo', 'aa', 'a+1']
 
 #### $a gives nothing
 declare -A a

spec/extended-glob.test.sh

@@ -70,104 +70,6 @@ argv.py @('__<>'|__{}|__\||__#|__&&)
 ## stdout: ['__<>', '__|', '__{}', '__&&', '__#']
 ## OK mksh stdout: ['__#', '__&&', '__<>', '__{}', '__|']
 
-#### @ matches exactly one
-[[ --verbose == --@(help|verbose) ]] && echo TRUE
-[[ --oops == --@(help|verbose) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### ? matches 0 or 1
-[[ -- == --?(help|verbose) ]] && echo TRUE
-[[ --oops == --?(help|verbose) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### + matches 1 or more
-[[ --helphelp == --+(help|verbose) ]] && echo TRUE
-[[ -- == --+(help|verbose) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### * matches 0 or more
-[[ -- == --*(help|verbose) ]] && echo TRUE
-[[ --oops == --*(help|verbose) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### simple repetition with *(foo) and +(Foo)
-[[ foofoo == *(foo) ]] && echo TRUE
-[[ foofoo == +(foo) ]] && echo TRUE
-## stdout-json: "TRUE\nTRUE\n"
-
-#### ! matches none
-[[ --oops == --!(help|verbose) ]] && echo TRUE
-[[ --help == --!(help|verbose) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### @() with variable arms
-choice1='help'
-choice2='verbose'
-[[ --verbose == --@($choice1|$choice2) ]] && echo TRUE
-[[ --oops == --@($choice1|$choice2) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### match is anchored
-[[ foo_ == @(foo) ]] || echo FALSE
-[[ _foo == @(foo) ]] || echo FALSE
-[[ foo == @(foo) ]] && echo TRUE
-## stdout-json: "FALSE\nFALSE\nTRUE\n"
-
-#### repeated match is anchored
-[[ foofoo_ == +(foo) ]] || echo FALSE
-[[ _foofoo == +(foo) ]] || echo FALSE
-[[ foofoo == +(foo) ]] && echo TRUE
-## stdout-json: "FALSE\nFALSE\nTRUE\n"
-
-#### repetition with glob
-# NOTE that * means two different things here
-[[ foofoo_foo__foo___ == *(foo*) ]] && echo TRUE
-[[ Xoofoo_foo__foo___ == *(foo*) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### No brace expansion in ==
-[[ --X{a,b}X == --@(help|X{a,b}X) ]] && echo TRUE
-[[ --oops == --@(help|X{a,b}X) ]] || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### adjacent extglob
-[[ --help == @(--|++)@(help|verbose) ]] && echo TRUE
-[[ ++verbose == @(--|++)@(help|verbose) ]] && echo TRUE
-## stdout-json: "TRUE\nTRUE\n"
-
-#### nested extglob
-[[ --help == --@(help|verbose=@(1|2)) ]] && echo TRUE
-[[ --verbose=1 == --@(help|verbose=@(1|2)) ]] && echo TRUE
-[[ --verbose=2 == --@(help|verbose=@(1|2)) ]] && echo TRUE
-[[ --verbose == --@(help|verbose=@(1|2)) ]] || echo FALSE
-## stdout-json: "TRUE\nTRUE\nTRUE\nFALSE\n"
-
-#### extglob in variable
-shopt -s extglob
-g=--@(help|verbose)
-quoted='--@(help|verbose)'
-[[ --help == $g ]] && echo TRUE
-[[ --verbose == $g ]] && echo TRUE
-[[ -- == $g ]] || echo FALSE
-[[ --help == $q ]] || echo FALSE
-[[ -- == $q ]] || echo FALSE
-## stdout-json: "TRUE\nTRUE\nFALSE\nFALSE\nFALSE\n"
-## N-I mksh stdout-json: "FALSE\nFALSE\nFALSE\n"
-
-#### extglob empty string
-shopt -s extglob
-[[ '' == @(foo|bar) ]] || echo FALSE
-[[ '' == @(foo||bar) ]] && echo TRUE
-## stdout-json: "FALSE\nTRUE\n"
-
-#### extglob empty pattern
-shopt -s extglob
-[[ '' == @() ]] && echo TRUE
-[[ '' == @(||) ]] && echo TRUE
-[[ X == @() ]] || echo FALSE
-[[ '|' == @(||) ]] || echo FALSE
-## stdout-json: "TRUE\nTRUE\nFALSE\nFALSE\n"
-
 #### printing extglob in variable
 # mksh does static parsing so it doesn't like this?
 shopt -s extglob
@@ -177,58 +79,3 @@ g=_tmp/eg3/@(foo|bar)
 echo $g "$g"  # quoting inhibits globbing
 ## stdout: _tmp/eg3/bar _tmp/eg3/foo _tmp/eg3/@(foo|bar)
 ## N-I mksh stdout: _tmp/eg3/@(foo|bar) _tmp/eg3/@(foo|bar)
-
-#### case with extglob
-shopt -s extglob
-for word in --help --verbose --unmatched -- -zxzx -; do
-  case $word in
-    --@(help|verbose) )
-      echo A
-      continue
-      ;;
-    ( --?(b|c) )
-      echo B
-      continue
-      ;;
-    ( -+(x|z) )
-      echo C
-      continue
-      ;;
-    ( -*(x|z) )
-      echo D
-      continue
-      ;;
-    *)
-      echo U
-      continue
-      ;;
-  esac
-done
-## stdout-json: "A\nA\nU\nB\nC\nD\n"
-
-#### Without shopt -s extglob
-empty=''
-str='x'
-[[ $empty == !($str) ]] && echo TRUE  # test glob match
-[[ $str == !($str) ]]   || echo FALSE
-## stdout-json: "TRUE\nFALSE\n"
-
-#### Turning extglob on changes the meaning of [[ !(str) ]] in bash
-empty=''
-str='x'
-[[ !($empty) ]]  && echo TRUE   # test if $empty is empty
-[[ !($str) ]]    || echo FALSE  # test if $str is empty
-shopt -s extglob  # mksh doesn't have this
-[[ !($empty) ]]  && echo TRUE   # negated glob
-[[ !($str) ]]    && echo TRUE   # negated glob
-## stdout-json: "TRUE\nFALSE\nTRUE\nTRUE\n"
-## OK mksh stdout-json: "TRUE\nTRUE\nTRUE\n"
-
-#### With extglob on, !($str) on the left or right of == has different meanings
-shopt -s extglob
-empty=''
-str='x'
-[[ 1 == !($str) ]]  && echo TRUE   # glob match
-[[ !($str) == 1 ]]  || echo FALSE  # test if empty
-# NOTE: There cannot be a space between ! and (?
-## stdout-json: "TRUE\nFALSE\n"