rgw: add an option to clear all usage entries

src/cls/rgw/cls_rgw.cc

@@ -3074,6 +3074,17 @@ int rgw_user_usage_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlis
   return 0;
 }
 
+int rgw_usage_log_clear(cls_method_context_t hctx, bufferlist *in, bufferlist *out)
+{
+  CLS_LOG(10,"%s", __func__);
+
+  int ret = cls_cxx_map_clear(hctx);
+  /* if object doesn't exist all the logs are cleared anyway */
+  if (ret == -ENOENT)
+    ret = 0;
+
+  return ret;
+}
 /*
  * We hold the garbage collection chain data under two different indexes: the first 'name' index
  * keeps them under a unique tag that represents the chains, and a second 'time' index keeps
@@ -3814,6 +3825,7 @@ CLS_INIT(rgw)
   cls_method_handle_t h_rgw_user_usage_log_add;
   cls_method_handle_t h_rgw_user_usage_log_read;
   cls_method_handle_t h_rgw_user_usage_log_trim;
+  cls_method_handle_t h_rgw_usage_log_clear;
   cls_method_handle_t h_rgw_gc_set_entry;
   cls_method_handle_t h_rgw_gc_list;
   cls_method_handle_t h_rgw_gc_remove;
@@ -3870,6 +3882,7 @@ CLS_INIT(rgw)
   cls_register_cxx_method(h_class, RGW_USER_USAGE_LOG_ADD, CLS_METHOD_RD | CLS_METHOD_WR, rgw_user_usage_log_add, &h_rgw_user_usage_log_add);
   cls_register_cxx_method(h_class, RGW_USER_USAGE_LOG_READ, CLS_METHOD_RD, rgw_user_usage_log_read, &h_rgw_user_usage_log_read);
   cls_register_cxx_method(h_class, RGW_USER_USAGE_LOG_TRIM, CLS_METHOD_RD | CLS_METHOD_WR, rgw_user_usage_log_trim, &h_rgw_user_usage_log_trim);
+  cls_register_cxx_method(h_class, RGW_USAGE_LOG_CLEAR, CLS_METHOD_WR, rgw_usage_log_clear, &h_rgw_usage_log_clear);
 
   /* garbage collection */
   cls_register_cxx_method(h_class, RGW_GC_SET_ENTRY, CLS_METHOD_RD | CLS_METHOD_WR, rgw_cls_gc_set_entry, &h_rgw_gc_set_entry);

src/cls/rgw/cls_rgw_client.cc

@@ -648,7 +648,11 @@ int cls_rgw_usage_log_trim(IoCtx& io_ctx, const string& oid, string& user,
   return 0;
 }
 
-
+void cls_rgw_usage_log_clear(ObjectWriteOperation& op)
+{
+  bufferlist in;
+  op.exec(RGW_CLASS, RGW_USAGE_LOG_CLEAR, in);
+}
 
 void cls_rgw_usage_log_add(ObjectWriteOperation& op, rgw_usage_log_info& info)
 {

src/cls/rgw/cls_rgw_client.h

@@ -506,6 +506,7 @@ int cls_rgw_usage_log_read(librados::IoCtx& io_ctx, string& oid, string& user,
 int cls_rgw_usage_log_trim(librados::IoCtx& io_ctx, const string& oid, string& user,
                            uint64_t start_epoch, uint64_t end_epoch);
 
+void cls_rgw_usage_log_clear(librados::ObjectWriteOperation& op);
 void cls_rgw_usage_log_add(librados::ObjectWriteOperation& op, rgw_usage_log_info& info);
 
 /* garbage collection */

src/cls/rgw/cls_rgw_const.h

@@ -39,6 +39,7 @@
 #define RGW_USER_USAGE_LOG_ADD "user_usage_log_add"
 #define RGW_USER_USAGE_LOG_READ "user_usage_log_read"
 #define RGW_USER_USAGE_LOG_TRIM "user_usage_log_trim"
+#define RGW_USAGE_LOG_CLEAR "usage_log_clear"
 
 /* garbage collection */
 #define RGW_GC_SET_ENTRY "gc_set_entry"

src/rgw/rgw_admin.cc

@@ -163,6 +163,7 @@ void usage()
   cout << "  log rm                     remove log object\n";
   cout << "  usage show                 show usage (by user, date range)\n";
   cout << "  usage trim                 trim usage (by user, date range)\n";
+  cout << "  usage clear                reset all the usage stats for the cluster\n";
   cout << "  gc list                    dump expired garbage collection objects (specify\n";
   cout << "                             --include-all to list all entries, including unexpired)\n";
   cout << "  gc process                 manually process garbage (specify\n";
@@ -378,6 +379,7 @@ enum {
   OPT_LOG_RM,
   OPT_USAGE_SHOW,
   OPT_USAGE_TRIM,
+  OPT_USAGE_CLEAR,
   OPT_OBJECT_RM,
   OPT_OBJECT_UNLINK,
   OPT_OBJECT_STAT,
@@ -647,6 +649,8 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_
       return OPT_USAGE_SHOW;
     if (strcmp(cmd, "trim") == 0)
       return OPT_USAGE_TRIM;
+    if (strcmp(cmd, "clear") == 0)
+      return OPT_USAGE_CLEAR;
   } else if (strcmp(prev_cmd, "caps") == 0) {
     if (strcmp(cmd, "add") == 0)
       return OPT_CAPS_ADD;
@@ -5273,6 +5277,20 @@ int main(int argc, const char **argv)
     }
   }
 
+  if (opt_cmd == OPT_USAGE_CLEAR) {
+    if (!yes_i_really_mean_it) {
+      cerr << "usage clear would remove *all* users usage data for all time" << std::endl;
+      cerr << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl;
+      return 1;
+    }
+
+    ret = RGWUsage::clear(store);
+    if (ret < 0) {
+      return ret;
+    }
+  }
+
+
   if (opt_cmd == OPT_OLH_GET || opt_cmd == OPT_OLH_READLOG) {
     if (bucket_name.empty()) {
       cerr << "ERROR: bucket not specified" << std::endl;

src/rgw/rgw_rados.cc

@@ -5165,6 +5165,22 @@ int RGWRados::trim_usage(rgw_user& user, uint64_t start_epoch, uint64_t end_epoc
   return 0;
 }
 
+
+int RGWRados::clear_usage()
+{
+  auto max_shards = cct->_conf->rgw_usage_max_shards;
+  int ret=0;
+  for (unsigned i=0; i < max_shards; i++){
+    string oid = RGW_USAGE_OBJ_PREFIX + to_string(i);
+    ret = cls_obj_usage_log_clear(oid);
+    if (ret < 0){
+      ldout(cct,0) << "usage clear on oid="<< oid << "failed with ret=" << ret << dendl;
+      return ret;
+    }
+  }
+  return ret;
+}
+
 int RGWRados::key_to_shard_id(const string& key, int max_shards)
 {
   return rgw_shard_id(key, max_shards);
@@ -12964,6 +12980,22 @@ int RGWRados::cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_e
   return r;
 }
 
+int RGWRados::cls_obj_usage_log_clear(string& oid)
+{
+  rgw_raw_obj obj(get_zone_params().usage_log_pool, oid);
+
+  rgw_rados_ref ref;
+  int r = get_raw_obj_ref(obj, &ref);
+  if (r < 0) {
+    return r;
+  }
+  librados::ObjectWriteOperation op;
+  cls_rgw_usage_log_clear(op);
+  r = ref.ioctx.operate(ref.oid, &op);
+  return r;
+}
+
+
 int RGWRados::remove_objs_from_index(RGWBucketInfo& bucket_info, list<rgw_obj_index_key>& oid_list)
 {
   librados::IoCtx index_ctx;

src/rgw/rgw_rados.h

@@ -2619,6 +2619,7 @@ class RGWRados
   int read_usage(const rgw_user& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
                  bool *is_truncated, RGWUsageIter& read_iter, map<rgw_user_bucket, rgw_usage_log_entry>& usage);
   int trim_usage(rgw_user& user, uint64_t start_epoch, uint64_t end_epoch);
+  int clear_usage();
 
   int create_pool(const rgw_pool& pool);
 
@@ -3486,6 +3487,7 @@ class RGWRados
   int cls_obj_usage_log_read(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch, uint32_t max_entries,
                              string& read_iter, map<rgw_user_bucket, rgw_usage_log_entry>& usage, bool *is_truncated);
   int cls_obj_usage_log_trim(string& oid, string& user, uint64_t start_epoch, uint64_t end_epoch);
+  int cls_obj_usage_log_clear(string& oid);
 
   int key_to_shard_id(const string& key, int max_shards);
   void shard_name(const string& prefix, unsigned max_shards, const string& key, string& name, int *shard_id);

src/rgw/rgw_usage.cc

@@ -145,3 +145,8 @@ int RGWUsage::trim(RGWRados *store, rgw_user& uid, uint64_t start_epoch,
 {
   return store->trim_usage(uid, start_epoch, end_epoch);
 }
+
+int RGWUsage::clear(RGWRados *store)
+{
+  return store->clear_usage();
+}

src/rgw/rgw_usage.h

@@ -23,6 +23,8 @@ class RGWUsage
 
   static int trim(RGWRados *store, rgw_user& uid, uint64_t start_epoch,
 	          uint64_t end_epoch);
+
+  static int clear(RGWRados *store);
 };
 
 

src/test/cli/radosgw-admin/help.t

@@ -106,6 +106,7 @@
     log rm                     remove log object
     usage show                 show usage (by user, date range)
     usage trim                 trim usage (by user, date range)
+    usage clear                reset all the usage stats for the cluster
     gc list                    dump expired garbage collection objects (specify
                                --include-all to list all entries, including unexpired)
     gc process                 manually process garbage (specify

src/test/cls_rgw/test_cls_rgw.cc

@@ -693,21 +693,28 @@ TEST(cls_rgw, gc_defer)
   ASSERT_EQ(0, destroy_one_pool_pp(gc_pool_name, rados));
 }
 
+auto populate_usage_log_info(std::string user, std::string payer, int total_usage_entries)
+{
+  rgw_usage_log_info info;
+
+  for (int i=0; i < total_usage_entries; i++){
+    auto bucket = str_int("bucket", i);
+    info.entries.emplace_back(rgw_usage_log_entry(user, payer, bucket));
+  }
+
+  return info;
+}
+
 TEST(cls_rgw, usage_basic)
 {
   string oid="usage.1";
   string user="user1";
   uint64_t start_epoch{0}, end_epoch{(uint64_t) -1};
-  constexpr auto total_usage_entries = 512;
+  int total_usage_entries = 512;
   uint64_t max_entries = 2000;
+  string payer;
 
-  rgw_usage_log_info info;
-
-  for (int i=0; i < total_usage_entries; i++){
-    auto bucket = str_int("bucket", i);
-    string p; // we are not testing bucket payer here
-    info.entries.emplace_back(rgw_usage_log_entry(user, p, bucket));
-  }
+  auto info = populate_usage_log_info(user, payer, total_usage_entries);
   ObjectWriteOperation op;
   cls_rgw_usage_log_add(op, info);
   ASSERT_EQ(0, ioctx.operate(oid, &op));
@@ -735,6 +742,47 @@ TEST(cls_rgw, usage_basic)
 
 }
 
+TEST(cls_rgw, usage_clear_no_obj)
+{
+  string user="user1";
+  string oid="usage.10";
+  librados::ObjectWriteOperation op;
+  cls_rgw_usage_log_clear(op);
+  int ret = ioctx.operate(oid, &op);
+  ASSERT_EQ(0, ret);
+
+}
+
+TEST(cls_rgw, usage_clear)
+{
+  string user="user1";
+  string payer;
+  string oid="usage.10";
+  librados::ObjectWriteOperation op;
+  int max_entries=2000;
+
+  auto info = populate_usage_log_info(user, payer, max_entries);
+
+  cls_rgw_usage_log_add(op, info);
+  ASSERT_EQ(0, ioctx.operate(oid, &op));
+
+  ObjectWriteOperation op2;
+  cls_rgw_usage_log_clear(op2);
+  int ret = ioctx.operate(oid, &op2);
+  ASSERT_EQ(0, ret);
+
+  map <rgw_user_bucket, rgw_usage_log_entry> usage;
+  bool truncated;
+  uint64_t start_epoch{0}, end_epoch{(uint64_t) -1};
+  string read_iter;
+  ret = cls_rgw_usage_log_read(ioctx, oid, user, start_epoch, end_epoch,
+			       max_entries, read_iter, usage, &truncated);
+  ASSERT_EQ(0, ret);
+  ASSERT_EQ(0, usage.size());
+
+}
+
+
 /* must be last test! */
 
 TEST(cls_rgw, finalize)