rgw: add support for "end_marker" on listing Swift account.

Fixes: #10682
Backport: hammer
Signed-off-by: Radoslaw Zarzynski <rzarzynski@mirantis.com>

src/cls/user/cls_user.cc

@@ -289,7 +289,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
 
   map<string, bufferlist> keys;
 
-  string from_index = op.marker;
+  const string from_index = op.marker;
+  const string to_index = op.end_marker;
+  const bool to_index_valid = !to_index.empty();
 
 #define MAX_ENTRIES 1000
   size_t max_entries = op.max_entries;
@@ -302,7 +304,10 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
   if (rc < 0)
     return rc;
 
-  CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str());
+  CLS_LOG(20, "from_index=%s to_index=%s match_prefix=%s",
+          from_index.c_str(),
+          to_index.c_str(),
+          match_prefix.c_str());
   cls_user_list_buckets_ret ret;
 
   list<cls_user_bucket_entry>& entries = ret.entries;
@@ -316,6 +321,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
     const string& index = iter->first;
     marker = index;
 
+    if (to_index_valid && to_index.compare(index) <= 0)
+      break;
+
     bufferlist& bl = iter->second;
     bufferlist::iterator biter = bl.begin();
     try {

src/cls/user/cls_user_client.cc

@@ -72,12 +72,18 @@ class ClsUserListCtx : public ObjectOperationCompletion {
 };
 
 void cls_user_bucket_list(librados::ObjectReadOperation& op,
-                       const string& in_marker, int max_entries, list<cls_user_bucket_entry>& entries,
-                       string *out_marker, bool *truncated, int *pret)
+                          const string& in_marker,
+                          string end_marker,
+                          int max_entries,
+                          list<cls_user_bucket_entry>& entries,
+                          string *out_marker,
+                          bool *truncated,
+                          int *pret)
 {
   bufferlist inbl;
   cls_user_list_buckets_op call;
   call.marker = in_marker;
+  call.end_marker = end_marker;
   call.max_entries = max_entries;
 
   ::encode(call, inbl);

src/cls/user/cls_user_client.h

@@ -23,9 +23,12 @@ void cls_user_set_buckets(librados::ObjectWriteOperation& op, list<cls_user_buck
 void cls_user_complete_stats_sync(librados::ObjectWriteOperation& op);
 void cls_user_remove_bucket(librados::ObjectWriteOperation& op,  const cls_user_bucket& bucket);
 void cls_user_bucket_list(librados::ObjectReadOperation& op,
-                       const string& in_marker, int max_entries,
+                       const string& in_marker,
+                       string end_marker,
+                       int max_entries,
                        list<cls_user_bucket_entry>& entries,
-                       string *out_marker, bool *truncated,
+                       string *out_marker,
+                       bool *truncated,
                        int *pret);
 void cls_user_get_header(librados::ObjectReadOperation& op, cls_user_header *header, int *pret);
 int cls_user_get_header_async(librados::IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx);

src/cls/user/cls_user_ops.h

@@ -59,6 +59,7 @@ WRITE_CLASS_ENCODER(cls_user_remove_bucket_op)
 
 struct cls_user_list_buckets_op {
   string marker;
+  string end_marker;
   int max_entries; /* upperbound to returned num of entries
                       might return less than that and still be truncated */
 
@@ -68,13 +69,15 @@ struct cls_user_list_buckets_op {
   void encode(bufferlist& bl) const {
     ENCODE_START(1, 1, bl);
     ::encode(marker, bl);
+    ::encode(end_marker, bl);
     ::encode(max_entries, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
     DECODE_START(1, bl);
     ::decode(marker, bl);
+    ::decode(end_marker, bl);
     ::decode(max_entries, bl);
     DECODE_FINISH(bl);
   }

src/rgw/rgw_bucket.cc

@@ -42,8 +42,13 @@ void rgw_get_buckets_obj(const string& user_id, string& buckets_obj_id)
  * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
  * Returns: 0 on success, -ERR# on failure.
  */
-int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
-                          const string& marker, uint64_t max, bool need_stats)
+int rgw_read_user_buckets(RGWRados *store,
+                          string user_id,
+                          RGWUserBuckets& buckets,
+                          const string& marker,
+                          uint64_t max,
+                          bool need_stats,
+                          string end_marker)
 {
   int ret;
   buckets.clear();
@@ -60,7 +65,7 @@ int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& bucke
   uint64_t total = 0;
 
   do {
-    ret = store->cls_user_list_buckets(obj, m, max - total, entries, &m, &truncated);
+    ret = store->cls_user_list_buckets(obj, m, max - total, entries, &m, &truncated, end_marker);
     if (ret == -ENOENT)
       ret = 0;
 

src/rgw/rgw_bucket.h

@@ -104,8 +104,13 @@ extern void rgw_bucket_init(RGWMetadataManager *mm);
  * Get all the buckets owned by a user and fill up an RGWUserBuckets with them.
  * Returns: 0 on success, -ERR# on failure.
  */
-extern int rgw_read_user_buckets(RGWRados *store, string user_id, RGWUserBuckets& buckets,
-                                 const string& marker, uint64_t max, bool need_stats);
+extern int rgw_read_user_buckets(RGWRados *store,
+                                 string user_id,
+                                 RGWUserBuckets& buckets,
+                                 const string& marker,
+                                 uint64_t max,
+                                 bool need_stats,
+                                 string end_marker = string());
 
 extern int rgw_link_bucket(RGWRados *store, string user_id, rgw_bucket& bucket, time_t creation_time, bool update_entrypoint = true);
 extern int rgw_unlink_bucket(RGWRados *store, string user_id, const string& bucket_name, bool update_entrypoint = true);

src/rgw/rgw_op.cc

@@ -1005,7 +1005,7 @@ void RGWListBuckets::execute()
       read_count = max_buckets;
 
     ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
-                                marker, read_count, should_get_stats());
+                                marker, read_count, should_get_stats(), end_marker);
 
     if (!started) {
       send_response_begin(buckets.count() > 0);

src/rgw/rgw_op.h

@@ -185,6 +185,7 @@ class RGWListBuckets : public RGWOp {
   int ret;
   bool sent_data;
   string marker;
+  string end_marker;
   uint64_t limit;
   uint64_t limit_max;
   uint32_t buckets_count;

src/rgw/rgw_rados.cc

@@ -8011,9 +8011,12 @@ int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket
 }
 
 int RGWRados::cls_user_list_buckets(rgw_obj& obj,
-                                    const string& in_marker, int max_entries,
+                                    const string& in_marker,
+                                    const int max_entries,
                                     list<cls_user_bucket_entry>& entries,
-                                    string *out_marker, bool *truncated)
+                                    string * const out_marker,
+                                    bool * const truncated,
+                                    string end_marker)
 {
   rgw_rados_ref ref;
   rgw_bucket bucket;
@@ -8025,7 +8028,7 @@ int RGWRados::cls_user_list_buckets(rgw_obj& obj,
   librados::ObjectReadOperation op;
   int rc;
 
-  cls_user_bucket_list(op, in_marker, max_entries, entries, out_marker, truncated, &rc);
+  cls_user_bucket_list(op, in_marker, end_marker, max_entries, entries, out_marker, truncated, &rc);
   bufferlist ibl;
   r = ref.ioctx.operate(ref.oid, &op, &ibl);
   if (r < 0)

src/rgw/rgw_rados.h

@@ -2025,9 +2025,12 @@ class RGWRados
   int cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket);
   int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
   int cls_user_list_buckets(rgw_obj& obj,
-                            const string& in_marker, int max_entries,
+                            const string& in_marker,
+                            int max_entries,
                             list<cls_user_bucket_entry>& entries,
-                            string *out_marker, bool *truncated);
+                            string *out_marker,
+                            bool *truncated,
+                            string end_marker = string());
   int cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry);
   int cls_user_update_buckets(rgw_obj& obj, list<cls_user_bucket_entry>& entries, bool add);
   int cls_user_complete_stats_sync(rgw_obj& obj);

src/rgw/rgw_rest_swift.cc

@@ -17,6 +17,7 @@
 int RGWListBuckets_ObjStore_SWIFT::get_params()
 {
   marker = s->info.args.get("marker");
+  end_marker = s->info.args.get("end_marker");
   string limit_str;
   limit_str = s->info.args.get("limit");
   long l = strtol(limit_str.c_str(), NULL, 10);