Permalink
Browse files

* S3/S3.py: Fix bucket listing for buckets with

  over 1000 prefixes. (contributed by Timothee Linden)



git-svn-id: https://s3tools.svn.sourceforge.net/svnroot/s3tools/s3cmd/trunk@413 830e0280-6d2a-0410-9c65-932aecc39d9d
  • Loading branch information...
1 parent 627aa58 commit 502817756d40fe13d1c2d508d89c96089281ac7f ludvigm committed May 26, 2010
Showing with 23 additions and 8 deletions.
  1. +5 −0 ChangeLog
  2. +18 −8 S3/S3.py
View
@@ -1,3 +1,8 @@
+2010-05-27 Michal Ludvig <mludvig@logix.net.nz>
+
+ * S3/S3.py: Fix bucket listing for buckets with
+ over 1000 prefixes. (contributed by Timothee Linden)
+
2010-05-21 Michal Ludvig <mludvig@logix.net.nz>
* s3cmd, S3/S3.py: Added support for bucket locations
View
@@ -178,15 +178,25 @@ def _get_common_prefixes(data):
return getListFromXml(data, "CommonPrefixes")
uri_params = {}
- response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)
- list = _get_contents(response["data"])
- prefixes = _get_common_prefixes(response["data"])
- while _list_truncated(response["data"]):
- uri_params['marker'] = self.urlencode_string(list[-1]["Key"])
- debug("Listing continues after '%s'" % uri_params['marker'])
+ truncated = True
+ list = []
+ prefixes = []
+
+ while truncated:
response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)
- list += _get_contents(response["data"])
- prefixes += _get_common_prefixes(response["data"])
+ curList = _get_contents(response["data"])
+ curPrefixes = _get_common_prefixes(response["data"])
+ truncated = _list_truncated(response["data"])
+ if truncated:
+ if curList:
+ uri_params['marker'] = self.urlencode_string( curList[-1]["Key"] )
+ else:
+ uri_params['marker'] = self.urlencode_string( curPrefixes[-1]["Prefix"] )
+ debug("Listing continues after '%s'" % uri_params['marker'])
+
+ list += curList
+ prefixes += curPrefixes
+
response['list'] = list
response['common_prefixes'] = prefixes
return response

0 comments on commit 5028177

Please sign in to comment.