Skip to content

Commit

Permalink
Explicitly disallow prefix recursion on objects with wildcards
Browse files Browse the repository at this point in the history
This fixes an infinite loop that can occur if a user runs
ls gs://bucket/object* on a bucket that contains an object named
"gs://bucket/object*/"
  • Loading branch information
thobrla committed Nov 2, 2015
1 parent 8b8d8ed commit 46c0995
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 0 deletions.
24 changes: 24 additions & 0 deletions gslib/tests/test_ls.py
Original file line number Diff line number Diff line change
Expand Up @@ -450,6 +450,30 @@ def test_recursive_list_trailing_two_slash(self):
# removed.
self.assertIn(suri(bucket_uri) + '//', stdout)

def test_wildcard_prefix(self):
"""Tests that an object name with a wildcard does not infinite loop."""
bucket_uri = self.CreateBucket()
wildcard_folder_object = 'wildcard*/'
object_matching_folder = 'wildcard10/foo'
self.CreateObject(bucket_uri=bucket_uri, object_name=wildcard_folder_object,
contents='foo')
self.CreateObject(bucket_uri=bucket_uri, object_name=object_matching_folder,
contents='foo')
self.AssertNObjectsInBucket(bucket_uri, 2)
stderr = self.RunGsUtil(['ls', suri(bucket_uri, 'wildcard*')],
return_stderr=True, expected_status=1)
self.assertIn('Cloud folder %s%s contains a wildcard' %
(suri(bucket_uri), '/wildcard*/'), stderr)

# Listing with a flat wildcard should still succeed.
# Use @Retry as hedge against bucket listing eventual consistency.
@Retry(AssertionError, tries=3, timeout_secs=1)
def _Check():
stdout = self.RunGsUtil(['ls', '-l', suri(bucket_uri, '**')],
return_stdout=True)
self.assertNumLines(stdout, 3) # 2 object lines, one summary line.
_Check()

@SkipForS3('S3 anonymous access is not supported.')
def test_get_object_without_list_bucket_permission(self):
# Bucket is not publicly readable by default.
Expand Down
11 changes: 11 additions & 0 deletions gslib/wildcard_iterator.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,17 @@ def __iter__(self, bucket_listing_fields=None,
self.all_versions or single_version_request))
else: # CloudApi.CsObjectOrPrefixType.PREFIX
prefix = obj_or_prefix.data

if ContainsWildcard(prefix):
# TODO: Disambiguate user-supplied strings from iterated
# prefix and object names so that we can better reason
# about wildcards and handle this case without raising an error.
raise CommandException(
'Cloud folder %s%s contains a wildcard; gsutil does '
'not currently support objects with wildcards in their '
'name.'
% (bucket_url_string, prefix))

# If the prefix ends with a slash, remove it. Note that we only
# remove one slash so that we can successfully enumerate dirs
# containing multiple slashes.
Expand Down

0 comments on commit 46c0995

Please sign in to comment.