add "limit" option to "ls" and "la" commands to return the specified

number of objects instead of returning all objects in the bucket.

S3/Config.py

@@ -76,6 +76,7 @@ class Config(object):
     delete_after = False
     delete_after_fetch = False
     max_delete = -1
+    limit = -1
     _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"
     delay_updates = False  # OBSOLETE
     gpg_passphrase = ""

S3/S3.py

@@ -292,19 +292,20 @@ def list_all_buckets(self):
         response["list"] = getListFromXml(response["data"], "Bucket")
         return response
 
-    def bucket_list(self, bucket, prefix = None, recursive = None, uri_params = {}):
+    def bucket_list(self, bucket, prefix = None, recursive = None, uri_params = {}, limit = -1):
         item_list = []
         prefixes = []
-        for dirs, objects in self.bucket_list_streaming(bucket, prefix, recursive, uri_params):
+        for truncated, dirs, objects in self.bucket_list_streaming(bucket, prefix, recursive, uri_params, limit):
             item_list.extend(objects)
             prefixes.extend(dirs)
 
         response = {}
         response['list'] = item_list
         response['common_prefixes'] = prefixes
+        response['truncated'] = truncated
         return response
 
-    def bucket_list_streaming(self, bucket, prefix = None, recursive = None, uri_params = {}):
+    def bucket_list_streaming(self, bucket, prefix = None, recursive = None, uri_params = {}, limit = -1):
         """ Generator that produces <dir_list>, <object_list> pairs of groups of content of a specified bucket. """
         def _list_truncated(data):
             ## <IsTruncated> can either be "true" or "false" or be missing completely
@@ -321,25 +322,38 @@ def _get_common_prefixes(data):
         truncated = True
         prefixes = []
 
+        num_objects = 0
+        num_prefixes = 0
+        max_keys = limit
         while truncated:
-            response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params)
+            response = self.bucket_list_noparse(bucket, prefix, recursive, uri_params, max_keys)
             current_list = _get_contents(response["data"])
             current_prefixes = _get_common_prefixes(response["data"])
+            num_objects += len(current_list)
+            num_prefixes += len(current_prefixes)
+            if limit > num_objects + num_prefixes:
+                max_keys = limit - (num_objects + num_prefixes)
             truncated = _list_truncated(response["data"])
             if truncated:
-                if current_list:
-                    uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"])
+                if limit == -1 or num_objects + num_prefixes < limit:
+                    if current_list:
+                        uri_params['marker'] = self.urlencode_string(current_list[-1]["Key"])
+                    else:
+                        uri_params['marker'] = self.urlencode_string(current_prefixes[-1]["Prefix"])
+                    debug("Listing continues after '%s'" % uri_params['marker'])
                 else:
-                    uri_params['marker'] = self.urlencode_string(current_prefixes[-1]["Prefix"])
-                debug("Listing continues after '%s'" % uri_params['marker'])
+                    yield truncated, current_prefixes, current_list
+                    break
 
-            yield current_prefixes, current_list
+            yield truncated, current_prefixes, current_list
 
-    def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}):
+    def bucket_list_noparse(self, bucket, prefix = None, recursive = None, uri_params = {}, max_keys = -1):
         if prefix:
             uri_params['prefix'] = self.urlencode_string(prefix)
         if not self.config.recursive and not recursive:
             uri_params['delimiter'] = "/"
+        if max_keys != -1:
+            uri_params['max-keys'] = str(max_keys)
         request = self.create_request("BUCKET_LIST", bucket = bucket, **uri_params)
         response = self.send_request(request)
         #debug(response)

s3cmd

@@ -117,7 +117,7 @@ def cmd_ls(args):
     if len(args) > 0:
         uri = S3Uri(args[0])
         if uri.type == "s3" and uri.has_bucket():
-            subcmd_bucket_list(s3, uri)
+            subcmd_bucket_list(s3, uri, cfg.limit)
             return EX_OK
 
     # If not a s3 type uri or no bucket was provided, list all the buckets
@@ -138,19 +138,19 @@ def cmd_all_buckets_list_all_content(args):
     response = s3.list_all_buckets()
 
     for bucket in response["list"]:
-        subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"]))
+        subcmd_bucket_list(s3, S3Uri("s3://" + bucket["Name"]), cfg.limit)
         output(u"")
     return EX_OK
 
-def subcmd_bucket_list(s3, uri):
+def subcmd_bucket_list(s3, uri, limit):
     bucket = uri.bucket()
     prefix = uri.object()
 
     debug(u"Bucket 's3://%s':" % bucket)
     if prefix.endswith('*'):
         prefix = prefix[:-1]
     try:
-        response = s3.bucket_list(bucket, prefix = prefix)
+        response = s3.bucket_list(bucket, prefix = prefix, limit = limit)
     except S3Error, e:
         if S3.codes.has_key(e.info["Code"]):
             error(S3.codes[e.info["Code"]] % bucket)
@@ -195,6 +195,9 @@ def subcmd_bucket_list(s3, uri):
             "uri": uri.compose_uri(bucket, object["Key"]),
             })
 
+    if response["truncated"]:
+        warning(u"The list is truncated because the settings limit was reached.")
+
 def cmd_bucket_create(args):
     s3 = S3(Config())
     for arg in args:
@@ -2531,6 +2534,7 @@ def main():
     optparser.add_option(      "--delete-after", dest="delete_after", action="store_true", help="Perform deletes after new uploads [sync]")
     optparser.add_option(      "--delay-updates", dest="delay_updates", action="store_true", help="*OBSOLETE* Put all updated files into place at end [sync]")  # OBSOLETE
     optparser.add_option(      "--max-delete", dest="max_delete", action="store", help="Do not delete more than NUM files. [del] and [sync]", metavar="NUM")
+    optparser.add_option(      "--limit", dest="limit", action="store", help="Limit number of objects returned in the response body (only for [ls] and [la] commands)", metavar="NUM")
     optparser.add_option(      "--add-destination", dest="additional_destinations", action="append", help="Additional destination for parallel uploads, in addition to last arg.  May be repeated.")
     optparser.add_option(      "--delete-after-fetch", dest="delete_after_fetch", action="store_true", help="Delete remote objects after fetching to local file (only for [get] and [sync] commands).")
     optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")