Implement 'shopt -s nullglob' and 'set -f '(noglob), with tests.

Details:

- Implement LooksLikeGlob() with unit tests

Other:

- Add a stub/test for 'shopt -s failglob too', but it's not
  implemented.
- Fix up comments in core/glob_.py

Addresses issue #26.

core/builtin.py

@@ -51,7 +51,7 @@
 EBuiltin = util.Enum('EBuiltin', """
 NONE READ ECHO SHIFT
 CD PUSHD POPD DIRS
-EXPORT UNSET SET
+EXPORT UNSET SET SHOPT
 TRAP UMASK
 EXIT SOURCE DOT EVAL EXEC WAIT JOBS 
 COMPLETE COMPGEN DEBUG_LINE
@@ -177,6 +177,8 @@ def Resolve(argv0):
 
   elif argv0 == "set":
     return EBuiltin.SET
+  elif argv0 == "shopt":
+    return EBuiltin.SHOPT
   elif argv0 == "unset":
     return EBuiltin.UNSET
   elif argv0 == "complete":
@@ -501,6 +503,7 @@ def AddOptionsToArgSpec(spec):
   spec.Option('n', 'noexec')
   spec.Option('u', 'nounset')
   spec.Option('x', 'xtrace')
+  spec.Option('f', 'noglob')
   spec.Option(None, 'pipefail')
 
   spec.Option(None, 'debug-completion')
@@ -584,6 +587,32 @@ def Set(argv, exec_opts, mem):
     exec_opts.strict_scope = True
 
 
+SHOPT_SPEC = _Register('shopt')
+SHOPT_SPEC.ShortFlag('-s')  # set
+SHOPT_SPEC.ShortFlag('-u')  # unset
+
+
+def Shopt(argv, exec_opts):
+  arg, i = SHOPT_SPEC.Parse(argv)
+  #log('%s', arg)
+
+  b = None
+  if arg.s:
+    b = True
+  elif arg.u:
+    b = False
+
+  if b is None:
+    raise NotImplementedError  # Display options
+
+  for opt_name in argv[i:]:
+    if opt_name not in ('nullglob', 'failglob'):
+      raise args.UsageError('shopt: Invalid option %r' % opt_name)
+    setattr(exec_opts, opt_name, b)
+
+  return 0
+
+
 UNSET_SPEC = _Register('unset')
 UNSET_SPEC.ShortFlag('-v')
 UNSET_SPEC.ShortFlag('-f')

core/cmd_exec.py

@@ -236,6 +236,9 @@ def _RunBuiltin(self, builtin_id, argv):
     elif builtin_id == EBuiltin.SET:
       status = builtin.Set(argv, self.exec_opts, self.mem)
 
+    elif builtin_id == EBuiltin.SHOPT:
+      status = builtin.Shopt(argv, self.exec_opts)
+
     elif builtin_id == EBuiltin.UNSET:
       status = builtin.Unset(argv, self.mem, self.funcs)
 

core/glob_.py

@@ -8,7 +8,7 @@
 from core.util import log
 
 
-def LooksLikeGlob():
+def LooksLikeGlob(s):
   """
   TODO: Reference lib/glob /   glob_pattern functions in bash
   grep glob_pattern lib/glob/*
@@ -21,7 +21,21 @@ def LooksLikeGlob():
 
   Still need this for slow path / fast path of prefix/suffix/patsub ops.
   """
-  pass
+  left_bracket = False
+  i = 0
+  n = len(s)
+  while i < n:
+    c = s[i]
+    if c == '\\':
+      i += 1
+    elif c == '*' or c == '?':
+      return True
+    elif c == '[':
+      left_bracket = True
+    elif c == ']' and left_bracket:
+      return True
+    i += 1
+  return False
 
 
 # Glob Helpers for WordParts.
@@ -41,10 +55,15 @@ def GlobEscape(s):
   return escaped
 
 
-# TODO: Can probably get rid of this, as long as you save the original word.
 def _GlobUnescape(s):  # used by cmd_exec
-  """
-  If there is no glob match, just unescape the string.
+  """Remove glob escaping from a string.
+  
+  Used when there is no glob match.
+  TODO: Can probably get rid of this, as long as you save the original word.
+
+  Complicated example: 'a*b'*.py, which will be escaped to a\*b*.py.  So in
+  word_eval _JoinElideEscape and EvalWordToString you have to build two
+  'parallel' strings -- one escaped and one not.
   """
   unescaped = ''
   i = 0
@@ -67,52 +86,52 @@ def _GlobUnescape(s):  # used by cmd_exec
 
 class Globber:
   def __init__(self, exec_opts):
-    # TODO: separate into set_opts.glob_opts, and sh_opts.glob_opts?  Only if
-    # other shels use the same options as bash though.
+    self.exec_opts = exec_opts
 
-    # NOTE: Bash also respects the GLOBIGNORE variable, but no other shells do.
-    # Could a default GLOBIGNORE to ignore flags on the file system be part of
-    # the security solution?  It doesn't seem totally sound.
-
-    self.noglob = False  # set -f
+    # NOTE: Bash also respects the GLOBIGNORE variable, but no other shells
+    # do.  Could a default GLOBIGNORE to ignore flags on the file system be
+    # part of the security solution?  It doesn't seem totally sound.
 
     # shopt: why the difference?  No command line switch I guess.
     self.dotglob = False  # dotfiles are matched
-    self.failglob = False  # no matches is an error
     self.globstar = False  # ** for directories
     # globasciiranges - ascii or unicode char classes (unicode by default)
     # nocaseglob
-    self.nullglob = False  # no matches evaluates to empty, otherwise
     # extglob: the !() syntax
 
     # TODO: Figure out which ones are in other shells, and only support those?
     # - Include globstar since I use it, and zsh has it.
 
   def Expand(self, arg):
-    # TODO: Only try to glob if there are any glob metacharacters.
-    # Or maybe it is a conservative "avoid glob" heuristic?
-    #
-    # Non-glob but with glob characters:
-    # echo ][
-    # echo []  # empty
-    # echo []LICENSE  # empty
-    # echo [L]ICENSE  # this one is good
-    # So yeah you need to test the validity somehow.
+    """Given a string that could be a glob, return a list of strings."""
+    # e.g. don't glob 'echo' because it doesn't look like a glob
+    if not LooksLikeGlob(arg):
+      u = _GlobUnescape(arg)
+      return [u]
+    if self.exec_opts.noglob:
+      return [arg]
 
     try:
       #g = glob.glob(arg)  # Bad Python glob
-      # PROBLEM: / is significant and can't be escaped!  Hav eto avoid globbing it.
+      # PROBLEM: / is significant and can't be escaped!  Have to avoid
+      # globbing it.
       g = libc.glob(arg)
     except Exception as e:
       # - [C\-D] is invalid in Python?  Regex compilation error.
       # - [:punct:] not supported
       print("Error expanding glob %r: %s" % (arg, e))
       raise
-    #print('G', arg, g)
+    #log('glob %r -> %r', arg, g)
 
-    #log('Globbing %s', arg)
     if g:
       return g
-    else:
-      u = _GlobUnescape(arg)
-      return [u]
+    else:  # Nothing matched
+      if self.exec_opts.failglob: 
+        # TODO: Make the command return status 1.
+        raise NotImplementedError
+      if self.exec_opts.nullglob: 
+        return []
+      else:
+        # Return the original string
+        u = _GlobUnescape(arg)
+        return [u]

core/glob_test.py

@@ -24,6 +24,34 @@ def testEscapeUnescape(self):
       self.assertEqual(e, esc(u))
       self.assertEqual(u, unesc(e))
 
+  def testLooksLikeGlob(self):
+    # The way to test bash behavior is:
+    #   $ shopt -s nullglob; argv [    # not a glob
+    #   $ shopt -s nullglob; argv []   # is a glob
+    #   $ shopt -s nullglob; argv [][  # is a glob
+    CASES = [
+        (r'[]', True),
+        (r'[][', True),
+        (r'][', False),  # no balanced pair
+        (r'\[]', False),  # no balanced pair
+        (r'[', False),  # no balanced pair
+        (r']', False),  # no balanced pair
+        (r'echo', False),
+        (r'status=0', False),
+
+        (r'*', True),
+        (r'\*', False),
+        (r'\*.sh', False),
+
+        ('\\', False),
+        ('*\\', True),
+
+        ('?', True),
+    ]
+    for pat, expected in CASES:
+      self.assertEqual(expected, glob_.LooksLikeGlob(pat),
+                       '%s: expected %r' % (pat, expected))
+
 
 if __name__ == '__main__':
   unittest.main()

core/state.py

@@ -75,6 +75,10 @@ def __init__(self):
     self.noexec = False  # -n
     self.debug_completion = False
 
+    # shopt -s / -u
+    self.nullglob = False 
+    self.failglob = False 
+
     # OSH-specific
     self.strict_arith = False  # e.g. $(( x )) where x doesn't look like integer
     self.strict_array = False  # ${a} not ${a[0]}, require double quotes, etc.

core/word_eval.py

@@ -8,7 +8,7 @@
 
 from core import braces
 from core import expr_eval
-from core.glob_ import Globber, GlobEscape
+from core import glob_
 from core.id_kind import Id, Kind, IdName, LookupKind
 from core import runtime
 from core import state
@@ -227,7 +227,7 @@ def _JoinElideEscape(frag_arrays, elide_empty, glob_escape):
           any_glob = True
         else:
           # "*.py" should be glob-escaped to \*.py
-          escaped_frags.append(GlobEscape(frag.s))
+          escaped_frags.append(glob_.GlobEscape(frag.s))
 
       arg_str = ''.join(escaped_frags)
       #log('ARG STR %s', arg_str)
@@ -873,7 +873,7 @@ def __init__(self, mem, exec_opts, part_ev):
     self.exec_opts = exec_opts
 
     self.part_ev = part_ev
-    self.globber = Globber(exec_opts)
+    self.globber = glob_.Globber(exec_opts)
 
   def _EvalParts(self, word, quoted=False):
     """Helper for EvalWordToAny and _EvalWordAndReframe.
@@ -914,7 +914,7 @@ def EvalWordToString(self, word, do_fnmatch=False, decay=False):
           if part_val.do_glob:
             strs.append(part_val.s)
           else:
-            strs.append(GlobEscape(part_val.s))
+            strs.append(glob_.GlobEscape(part_val.s))
         else:
           strs.append(part_val.s)
 

spec/glob.test.sh

@@ -143,6 +143,28 @@ cd _tmp
 echo -* hello zzzz?
 # stdout-json: "hello zzzzz"
 
+### set -o noglob
+touch _tmp/spec-tmp/a.zz _tmp/spec-tmp/b.zz
+echo _tmp/spec-tmp/*.zz
+set -o noglob
+echo _tmp/spec-tmp/*.zz
+# stdout-json: "_tmp/spec-tmp/a.zz _tmp/spec-tmp/b.zz\n_tmp/spec-tmp/*.zz\n"
+
+### shopt -s nullglob
+argv.py _tmp/spec-tmp/*.nonexistent
+shopt -s nullglob
+argv.py _tmp/spec-tmp/*.nonexistent
+# stdout-json: "['_tmp/spec-tmp/*.nonexistent']\n[]\n"
+# N-I dash/mksh/ash stdout-json: "['_tmp/spec-tmp/*.nonexistent']\n['_tmp/spec-tmp/*.nonexistent']\n"
+
+### shopt -s failglob
+argv.py *.ZZ
+shopt -s failglob
+argv.py *.ZZ  # nothing is printed, not []
+echo status=$?
+# stdout-json: "['*.ZZ']\nstatus=1\n"
+# N-I dash/mksh/ash stdout-json: "['*.ZZ']\n['*.ZZ']\nstatus=0\n"
+
 ### Don't glob flags on file system with GLOBIGNORE
 # This is a bash-specific extension.
 expr $0 : '.*/osh$' >/dev/null && exit 99  # disabled until cd implemented

test/spec.sh

@@ -271,7 +271,7 @@ func() {
 }
 
 glob() {
-  sh-spec spec/glob.test.sh --osh-failures-allowed 1 \
+  sh-spec spec/glob.test.sh --osh-failures-allowed 2 \
     ${REF_SHELLS[@]} $BUSYBOX_ASH $OSH "$@"
 }