Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rgw: add bucket size limit check to radosgw-admin #11796

Merged
merged 1 commit into from Apr 20, 2017
Merged

rgw: add bucket size limit check to radosgw-admin #11796

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 4 additions & 0 deletions src/common/config_opts.h
View file
Open in desktop
Expand Up @@ -1651,6 +1651,10 @@ OPTION(rgw_realm_reconfigure_delay, OPT_DOUBLE, 2) // seconds to wait before rel
OPTION(rgw_period_push_interval, OPT_DOUBLE, 2) // seconds to wait before retrying "period push"
OPTION(rgw_period_push_interval_max, OPT_DOUBLE, 30) // maximum interval after exponential backoff

OPTION(rgw_safe_max_objects_per_shard, OPT_INT, 100*1024) // safe max loading
OPTION(rgw_shard_warning_threshold, OPT_DOUBLE, 90) // pct of safe max
// at which to warn

OPTION(rgw_swift_versioning_enabled, OPT_BOOL, false) // whether swift object versioning feature is enabled

OPTION(mgr_module_path, OPT_STR, CEPH_PKGLIBDIR "/mgr") // where to load python modules from
Expand Down
82 changes: 72 additions & 10 deletions src/rgw/rgw_admin.cc
View file
Open in desktop
@@ -1,3 +1,4 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab

#include <errno.h>
Expand Down Expand Up @@ -72,6 +73,7 @@ void _usage()
cout << " key create create access key\n";
cout << " key rm remove access key\n";
cout << " bucket list list buckets\n";
cout << " bucket limit check show bucket sharding stats\n";
cout << " bucket link link bucket to specified user\n";
cout << " bucket unlink unlink bucket from specified user\n";
cout << " bucket stats returns bucket statistics\n";
Expand Down Expand Up @@ -285,6 +287,9 @@ void _usage()
cout << " --categories=<list> comma separated list of categories, used in usage show\n";
cout << " --caps=<caps> list of caps (e.g., \"usage=read, write; user=read\")\n";
cout << " --yes-i-really-mean-it required for certain operations\n";
cout << " --warnings-only when specified with bucket limit check, list\n";
cout << " only buckets nearing or over the current max\n";
cout << " objects per shard value\n";
cout << " --bypass-gc when specified with bucket deletion, triggers\n";
cout << " object deletions by not involving GC\n";
cout << " --inconsistent-index when specified with bucket deletion and bypass-gc set to true,\n";
Expand Down Expand Up @@ -338,6 +343,7 @@ enum {
OPT_KEY_CREATE,
OPT_KEY_RM,
OPT_BUCKETS_LIST,
OPT_BUCKETS_LIMIT_CHECK,
OPT_BUCKET_LINK,
OPT_BUCKET_UNLINK,
OPT_BUCKET_STATS,
Expand Down Expand Up @@ -555,6 +561,10 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_
} else if (strcmp(prev_cmd, "buckets") == 0) {
if (strcmp(cmd, "list") == 0)
return OPT_BUCKETS_LIST;
if (strcmp(cmd, "limit") == 0) {
*need_more = true;
return 0;
}
} else if (strcmp(prev_cmd, "bucket") == 0) {
if (strcmp(cmd, "list") == 0)
return OPT_BUCKETS_LIST;
Expand All @@ -576,14 +586,18 @@ static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_
*need_more = true;
return 0;
}
} else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "bucket") == 0) &&
(strcmp(prev_cmd, "sync") == 0)) {
if (strcmp(cmd, "status") == 0)
return OPT_BUCKET_SYNC_STATUS;
if (strcmp(cmd, "init") == 0)
return OPT_BUCKET_SYNC_INIT;
if (strcmp(cmd, "run") == 0)
return OPT_BUCKET_SYNC_RUN;
} else if (prev_prev_cmd && strcmp(prev_prev_cmd, "bucket") == 0) {
if (strcmp(prev_cmd, "sync") == 0) {
if (strcmp(cmd, "status") == 0)
return OPT_BUCKET_SYNC_STATUS;
if (strcmp(cmd, "init") == 0)
return OPT_BUCKET_SYNC_INIT;
if (strcmp(cmd, "run") == 0)
return OPT_BUCKET_SYNC_RUN;
} else if ((strcmp(prev_cmd, "limit") == 0) &&
(strcmp(cmd, "check") == 0)) {
return OPT_BUCKETS_LIMIT_CHECK;
}
} else if (strcmp(prev_cmd, "log") == 0) {
if (strcmp(cmd, "list") == 0)
return OPT_LOG_LIST;
Expand Down Expand Up @@ -2437,6 +2451,7 @@ int main(int argc, const char **argv)

int sync_stats = false;
int bypass_gc = false;
int warnings_only = false;
int inconsistent_index = false;

int verbose = false;
Expand Down Expand Up @@ -2674,6 +2689,8 @@ int main(int argc, const char **argv)
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &bypass_gc, NULL, "--bypass-gc", (char*)NULL)) {
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &warnings_only, NULL, "--warnings-only", (char*)NULL)) {
// do nothing
} else if (ceph_argparse_binary_flag(args, i, &inconsistent_index, NULL, "--inconsistent-index", (char*)NULL)) {
// do nothing
} else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
Expand Down Expand Up @@ -4812,6 +4829,51 @@ int main(int argc, const char **argv)
}
}

if (opt_cmd == OPT_BUCKETS_LIMIT_CHECK) {
void *handle;
std::list<std::string> user_ids;
metadata_key = "user";
int max = 1000;

bool truncated;

if (! user_id.empty()) {
user_ids.push_back(user_id.id);
ret =
RGWBucketAdminOp::limit_check(store, bucket_op, user_ids, f,
warnings_only);
} else {
/* list users in groups of max-keys, then perform user-bucket
* limit-check on each group */
ret = store->meta_mgr->list_keys_init(metadata_key, &handle);
if (ret < 0) {
cerr << "ERROR: buckets limit check can't get user metadata_key: "
<< cpp_strerror(-ret) << std::endl;
return -ret;
}

do {
ret = store->meta_mgr->list_keys_next(handle, max, user_ids,
&truncated);
if (ret < 0 && ret != -ENOENT) {
cerr << "ERROR: buckets limit check lists_keys_next(): "
<< cpp_strerror(-ret) << std::endl;
break;
} else {
/* ok, do the limit checks for this group */
ret =
RGWBucketAdminOp::limit_check(store, bucket_op, user_ids, f,
warnings_only);
if (ret < 0)
break;
}
user_ids.clear();
} while (truncated);
store->meta_mgr->list_keys_complete(handle);
}
return -ret;
} /* OPT_BUCKETS_LIMIT_CHECK */

if (opt_cmd == OPT_BUCKETS_LIST) {
if (bucket_name.empty()) {
RGWBucketAdminOp::info(store, bucket_op, f);
Expand Down Expand Up @@ -4862,8 +4924,8 @@ int main(int argc, const char **argv)

formatter->close_section();
formatter->flush(cout);
}
}
} /* have bucket_name */
} /* OPT_BUCKETS_LIST */

if (opt_cmd == OPT_BUCKET_STATS) {
bucket_op.set_fetch_stats(true);
Expand Down
124 changes: 123 additions & 1 deletion src/rgw/rgw_bucket.cc
View file
Open in desktop
Expand Up @@ -5,8 +5,10 @@

#include <string>
#include <map>
#include <sstream>

#include <boost/utility/string_ref.hpp>
#include <boost/format.hpp>

#include "common/errno.h"
#include "common/ceph_json.h"
Expand Down Expand Up @@ -1401,6 +1403,126 @@ static int bucket_stats(RGWRados *store, const std::string& tenant_name, std::st
return 0;
}

int RGWBucketAdminOp::limit_check(RGWRados *store,
RGWBucketAdminOpState& op_state,
const std::list<std::string>& user_ids,
RGWFormatterFlusher& flusher,
bool warnings_only)
{
int ret = 0;
const size_t max_entries =
store->ctx()->_conf->rgw_list_buckets_max_chunk;

const size_t safe_max_objs_per_shard =
store->ctx()->_conf->rgw_safe_max_objects_per_shard;

uint16_t shard_warn_pct =
store->ctx()->_conf->rgw_shard_warning_threshold;
if (shard_warn_pct > 100)
shard_warn_pct = 90;

Formatter *formatter = flusher.get_formatter();
flusher.start(0);

formatter->open_array_section("users");

for (const auto& user_id : user_ids) {
formatter->open_object_section("user");
formatter->dump_string("user_id", user_id);
bool done;
formatter->open_array_section("buckets");
do {
RGWUserBuckets buckets;
string marker;
bool is_truncated;

ret = rgw_read_user_buckets(store, user_id, buckets,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mattbenjamin need to make sure that this works correctly with users under non-default tenants

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

marker, string(), max_entries, false,
&is_truncated);
if (ret < 0)
return ret;

map<string, RGWBucketEnt>& m_buckets = buckets.get_buckets();

for (const auto& iter : m_buckets) {
auto& bucket = iter.second.bucket;
uint32_t num_shards = 1;
uint64_t num_objects = 0;

/* need info for num_shards */
RGWBucketInfo info;
RGWObjectCtx obj_ctx(store);

marker = bucket.name; /* Casey's location for marker update,
* as we may now not reach the end of
* the loop body */

ret = store->get_bucket_info(obj_ctx, bucket.tenant, bucket.name,
info, nullptr);
if (ret < 0)
continue;

/* need stats for num_entries */
string bucket_ver, master_ver;
std::map<RGWObjCategory, RGWStorageStats> stats;
ret = store->get_bucket_stats(info, RGW_NO_SHARD, &bucket_ver,
&master_ver, stats, nullptr);

if (ret < 0)
continue;

for (const auto& s : stats) {
num_objects += s.second.num_objects;
}

num_shards = info.num_shards;
uint64_t objs_per_shard = num_objects / num_shards;
{
bool warn = false;
stringstream ss;
if (objs_per_shard > safe_max_objs_per_shard) {
double over =
100 - (safe_max_objs_per_shard/objs_per_shard * 100);
ss << boost::format("OVER %4f%%") % over;
warn = true;
} else {
double fill_pct =
objs_per_shard / safe_max_objs_per_shard * 100;
if (fill_pct >= shard_warn_pct) {
ss << boost::format("WARN %4f%%") % fill_pct;
warn = true;
} else {
ss << "OK";
}
}

if (warn || (! warnings_only)) {
formatter->open_object_section("bucket");
formatter->dump_string("bucket", bucket.name);
formatter->dump_string("tenant", bucket.tenant);
formatter->dump_int("num_objects", num_objects);
formatter->dump_int("num_shards", num_shards);
formatter->dump_int("objects_per_shard", objs_per_shard);
formatter->dump_string("fill_status", ss.str());
formatter->close_section();
}
}
}

done = (m_buckets.size() < max_entries);
} while (!done); /* foreach: bucket */

formatter->close_section();
formatter->close_section();
formatter->flush(cout);

} /* foreach: user_id */

formatter->close_section();
formatter->flush(cout);

return ret;
} /* RGWBucketAdminOp::limit_check */

int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,
RGWFormatterFlusher& flusher)
Expand All @@ -1421,7 +1543,7 @@ int RGWBucketAdminOp::info(RGWRados *store, RGWBucketAdminOpState& op_state,

CephContext *cct = store->ctx();

size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;
const size_t max_entries = cct->_conf->rgw_list_buckets_max_chunk;

bool show_stats = op_state.will_fetch_stats();
rgw_user user_id = op_state.get_user_id();
Expand Down
4 changes: 4 additions & 0 deletions src/rgw/rgw_bucket.h
View file
Open in desktop
Expand Up @@ -315,6 +315,10 @@ class RGWBucketAdminOp
static int remove_bucket(RGWRados *store, RGWBucketAdminOpState& op_state, bool bypass_gc = false, bool keep_index_consistent = true);
static int remove_object(RGWRados *store, RGWBucketAdminOpState& op_state);
static int info(RGWRados *store, RGWBucketAdminOpState& op_state, RGWFormatterFlusher& flusher);
static int limit_check(RGWRados *store, RGWBucketAdminOpState& op_state,
const std::list<std::string>& user_ids,
RGWFormatterFlusher& flusher,
bool warnings_only = false);
};


Expand Down
4 changes: 4 additions & 0 deletions src/test/cli/radosgw-admin/help.t
View file
Open in desktop
Expand Up @@ -18,6 +18,7 @@
key create create access key
key rm remove access key
bucket list list buckets
bucket limit check show bucket sharding stats
bucket link link bucket to specified user
bucket unlink unlink bucket from specified user
bucket stats returns bucket statistics
Expand Down Expand Up @@ -231,6 +232,9 @@
--categories=<list> comma separated list of categories, used in usage show
--caps=<caps> list of caps (e.g., "usage=read, write; user=read")
--yes-i-really-mean-it required for certain operations
--warnings-only when specified with bucket limit check, list
only buckets nearing or over the current max
objects per shard value
--bypass-gc when specified with bucket deletion, triggers
object deletions by not involving GC
--inconsistent-index when specified with bucket deletion and bypass-gc set to true,
Expand Down