Merge pull request #4216 from rzarzynski/wip-10682

rgw: add support for "end_marker" parameter for GET on Swift account.

Reviewed-by: Yehuda Sadeh <yehuda@redhat.com>

src/cls/user/cls_user.cc

@@ -289,7 +289,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
 
   map<string, bufferlist> keys;
 
-  string from_index = op.marker;
+  const string& from_index = op.marker;
+  const string& to_index = op.end_marker;
+  const bool to_index_valid = !to_index.empty();
 
 #define MAX_ENTRIES 1000
   size_t max_entries = op.max_entries;
@@ -302,7 +304,10 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
   if (rc < 0)
     return rc;
 
-  CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str());
+  CLS_LOG(20, "from_index=%s to_index=%s match_prefix=%s",
+          from_index.c_str(),
+          to_index.c_str(),
+          match_prefix.c_str());
   cls_user_list_buckets_ret ret;
 
   list<cls_user_bucket_entry>& entries = ret.entries;
@@ -316,6 +321,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff
     const string& index = iter->first;
     marker = index;
 
+    if (to_index_valid && to_index.compare(index) <= 0)
+      break;
+
     bufferlist& bl = iter->second;
     bufferlist::iterator biter = bl.begin();
     try {

src/cls/user/cls_user_client.cc

@@ -72,12 +72,18 @@ class ClsUserListCtx : public ObjectOperationCompletion {
 };
 
 void cls_user_bucket_list(librados::ObjectReadOperation& op,
-                       const string& in_marker, int max_entries, list<cls_user_bucket_entry>& entries,
-                       string *out_marker, bool *truncated, int *pret)
+                          const string& in_marker,
+                          const string& end_marker,
+                          int max_entries,
+                          list<cls_user_bucket_entry>& entries,
+                          string *out_marker,
+                          bool *truncated,
+                          int *pret)
 {
   bufferlist inbl;
   cls_user_list_buckets_op call;
   call.marker = in_marker;
+  call.end_marker = end_marker;
   call.max_entries = max_entries;
 
   ::encode(call, inbl);

src/cls/user/cls_user_client.h

@@ -23,9 +23,12 @@ void cls_user_set_buckets(librados::ObjectWriteOperation& op, list<cls_user_buck
 void cls_user_complete_stats_sync(librados::ObjectWriteOperation& op);
 void cls_user_remove_bucket(librados::ObjectWriteOperation& op,  const cls_user_bucket& bucket);
 void cls_user_bucket_list(librados::ObjectReadOperation& op,
-                       const string& in_marker, int max_entries,
+                       const string& in_marker,
+                       const string& end_marker,
+                       int max_entries,
                        list<cls_user_bucket_entry>& entries,
-                       string *out_marker, bool *truncated,
+                       string *out_marker,
+                       bool *truncated,
                        int *pret);
 void cls_user_get_header(librados::ObjectReadOperation& op, cls_user_header *header, int *pret);
 int cls_user_get_header_async(librados::IoCtx& io_ctx, string& oid, RGWGetUserHeader_CB *ctx);

src/cls/user/cls_user_ops.h

@@ -59,23 +59,28 @@ WRITE_CLASS_ENCODER(cls_user_remove_bucket_op)
 
 struct cls_user_list_buckets_op {
   string marker;
+  string end_marker;
   int max_entries; /* upperbound to returned num of entries
                       might return less than that and still be truncated */
 
   cls_user_list_buckets_op()
     : max_entries(0) {}
 
   void encode(bufferlist& bl) const {
-    ENCODE_START(1, 1, bl);
+    ENCODE_START(2, 1, bl);
     ::encode(marker, bl);
     ::encode(max_entries, bl);
+    ::encode(end_marker, bl);
     ENCODE_FINISH(bl);
   }
 
   void decode(bufferlist::iterator& bl) {
-    DECODE_START(1, bl);
+    DECODE_START(2, bl);
     ::decode(marker, bl);
     ::decode(max_entries, bl);
+    if (struct_v >= 2) {
+      ::decode(end_marker, bl);
+    }
     DECODE_FINISH(bl);
   }
 

src/rgw/rgw_bucket.cc

@@ -88,6 +88,7 @@ int rgw_read_user_buckets(RGWRados * store,
                           const rgw_user& user_id,
                           RGWUserBuckets& buckets,
                           const string& marker,
+                          const string& end_marker,
                           uint64_t max,
                           bool need_stats,
                           uint64_t default_amount)
@@ -111,7 +112,7 @@ int rgw_read_user_buckets(RGWRados * store,
   }
 
   do {
-    ret = store->cls_user_list_buckets(obj, m, max - total, entries, &m, &truncated);
+    ret = store->cls_user_list_buckets(obj, m, end_marker, max - total, entries, &m, &truncated);
     if (ret == -ENOENT)
       ret = 0;
 
@@ -367,7 +368,8 @@ void check_bad_user_bucket_mapping(RGWRados *store, const rgw_user& user_id, boo
   size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
 
   do {
-    int ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, max_entries, false);
+    int ret = rgw_read_user_buckets(store, user_id, user_buckets,
+                                    marker, string(), max_entries, false);
     if (ret < 0) {
       ldout(store->ctx(), 0) << "failed to read user buckets: " << cpp_strerror(-ret) << dendl;
       return;
@@ -1126,7 +1128,8 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
     bool done;
 
     do {
-      ret = rgw_read_user_buckets(store, user_id, buckets, marker, max_entries, false);
+      ret = rgw_read_user_buckets(store, user_id, buckets,
+                                  marker, string(), max_entries, false);
       if (ret < 0)
         return ret;
 

src/rgw/rgw_bucket.h

@@ -119,6 +119,7 @@ extern int rgw_read_user_buckets(RGWRados *store,
                                  const rgw_user& user_id,
                                  RGWUserBuckets& buckets,
                                  const string& marker,
+                                 const string& end_marker,
                                  uint64_t max,
                                  bool need_stats,
                                  uint64_t default_amount = 1000);

src/rgw/rgw_op.cc

@@ -1312,7 +1312,7 @@ void RGWListBuckets::execute()
     }
 
     ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
-                                marker, read_count, should_get_stats(), 0);
+                                marker, end_marker, read_count, should_get_stats(), 0);
 
     if (ret < 0) {
       /* hmm.. something wrong here.. the user was authenticated, so it
@@ -1369,7 +1369,8 @@ void RGWStatAccount::execute()
   do {
     RGWUserBuckets buckets;
 
-    ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, max_buckets, false);
+    ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
+                                marker, string(), max_buckets, false);
     if (ret < 0) {
       /* hmm.. something wrong here.. the user was authenticated, so it
          should exist */
@@ -1571,7 +1572,8 @@ int RGWCreateBucket::verify_permission()
   if (s->user.max_buckets) {
     RGWUserBuckets buckets;
     string marker;
-    int ret = rgw_read_user_buckets(store, s->user.user_id, buckets, marker, s->user.max_buckets, false);
+    int ret = rgw_read_user_buckets(store, s->user.user_id, buckets,
+                                    marker, string(), s->user.max_buckets, false);
     if (ret < 0)
       return ret;
 

src/rgw/rgw_op.h

@@ -285,7 +285,8 @@ class RGWListBuckets : public RGWOp {
   int ret;
   bool sent_data;
   string marker;
-  int64_t limit;
+  string end_marker;
+  uint64_t limit;
   uint64_t limit_max;
   uint32_t buckets_count;
   uint64_t buckets_objcount;

src/rgw/rgw_rados.cc

@@ -8767,9 +8767,12 @@ int RGWRados::update_user_bucket_stats(const string& user_id, rgw_bucket& bucket
 }
 
 int RGWRados::cls_user_list_buckets(rgw_obj& obj,
-                                    const string& in_marker, int max_entries,
+                                    const string& in_marker,
+                                    const string& end_marker,
+                                    const int max_entries,
                                     list<cls_user_bucket_entry>& entries,
-                                    string *out_marker, bool *truncated)
+                                    string * const out_marker,
+                                    bool * const truncated)
 {
   rgw_rados_ref ref;
   rgw_bucket bucket;
@@ -8781,7 +8784,7 @@ int RGWRados::cls_user_list_buckets(rgw_obj& obj,
   librados::ObjectReadOperation op;
   int rc;
 
-  cls_user_bucket_list(op, in_marker, max_entries, entries, out_marker, truncated, &rc);
+  cls_user_bucket_list(op, in_marker, end_marker, max_entries, entries, out_marker, truncated, &rc);
   bufferlist ibl;
   r = ref.ioctx.operate(ref.oid, &op, &ibl);
   if (r < 0)

src/rgw/rgw_rados.h

@@ -2182,9 +2182,12 @@ class RGWRados
   int cls_user_sync_bucket_stats(rgw_obj& user_obj, rgw_bucket& bucket);
   int update_user_bucket_stats(const string& user_id, rgw_bucket& bucket, RGWStorageStats& stats);
   int cls_user_list_buckets(rgw_obj& obj,
-                            const string& in_marker, int max_entries,
+                            const string& in_marker,
+                            const string& end_marker,
+                            int max_entries,
                             list<cls_user_bucket_entry>& entries,
-                            string *out_marker, bool *truncated);
+                            string *out_marker,
+                            bool *truncated);
   int cls_user_add_bucket(rgw_obj& obj, const cls_user_bucket_entry& entry);
   int cls_user_update_buckets(rgw_obj& obj, list<cls_user_bucket_entry>& entries, bool add);
   int cls_user_complete_stats_sync(rgw_obj& obj);

src/rgw/rgw_rest_swift.cc

@@ -17,6 +17,7 @@
 int RGWListBuckets_ObjStore_SWIFT::get_params()
 {
   marker = s->info.args.get("marker");
+  end_marker = s->info.args.get("end_marker");
 
   string limit_str = s->info.args.get("limit");
   if (!limit_str.empty()) {

src/rgw/rgw_user.cc

@@ -55,7 +55,8 @@ int rgw_user_sync_all_stats(RGWRados *store, const rgw_user& user_id)
 
   do {
     RGWUserBuckets user_buckets;
-    ret = rgw_read_user_buckets(store, user_id, user_buckets, marker, max_entries, false);
+    ret = rgw_read_user_buckets(store, user_id, user_buckets,
+                                marker, string(), max_entries, false);
     if (ret < 0) {
       ldout(cct, 0) << "failed to read user buckets: ret=" << ret << dendl;
       return ret;
@@ -413,7 +414,8 @@ int rgw_delete_user(RGWRados *store, RGWUserInfo& info, RGWObjVersionTracker& ob
 
   do {
     RGWUserBuckets user_buckets;
-    ret = rgw_read_user_buckets(store, info.user_id, user_buckets, marker, max_buckets, false);
+    ret = rgw_read_user_buckets(store, info.user_id, user_buckets,
+                                marker, string(), max_buckets, false);
     if (ret < 0)
       return ret;
 
@@ -1970,7 +1972,8 @@ int RGWUser::execute_remove(RGWUserAdminOpState& op_state, std::string *err_msg)
   size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
   do {
     RGWUserBuckets buckets;
-    ret = rgw_read_user_buckets(store, uid, buckets, marker, max_buckets, false);
+    ret = rgw_read_user_buckets(store, uid, buckets,
+                                marker, string(), max_buckets, false);
     if (ret < 0) {
       set_err_msg(err_msg, "unable to read user bucket info");
       return ret;
@@ -2119,7 +2122,8 @@ int RGWUser::execute_modify(RGWUserAdminOpState& op_state, std::string *err_msg)
     CephContext *cct = store->ctx();
     size_t max_buckets = cct->_conf->rgw_list_buckets_max_chunk;
     do {
-      ret = rgw_read_user_buckets(store, user_id, buckets, marker, max_buckets, false);
+      ret = rgw_read_user_buckets(store, user_id, buckets,
+                                  marker, string(), max_buckets, false);
       if (ret < 0) {
         set_err_msg(err_msg, "could not get buckets for uid:  " + user_id.to_str());
         return ret;