diff --git a/src/cls/journal/cls_journal.cc b/src/cls/journal/cls_journal.cc index ffa18c3c56f7a..f966c07f24aed 100644 --- a/src/cls/journal/cls_journal.cc +++ b/src/cls/journal/cls_journal.cc @@ -101,13 +101,13 @@ int expire_tags(cls_method_context_t hctx, const std::string *skip_client_id) { skip_client_key = key_from_client_id(*skip_client_id); } - int r; uint64_t minimum_tag_tid = std::numeric_limits::max(); std::string last_read = HEADER_KEY_CLIENT_PREFIX; + bool more; do { std::map vals; - r = cls_cxx_map_get_vals(hctx, last_read, HEADER_KEY_CLIENT_PREFIX, - MAX_KEYS_READ, &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, HEADER_KEY_CLIENT_PREFIX, + MAX_KEYS_READ, &vals, &more); if (r < 0 && r != -ENOENT) { CLS_ERR("failed to retrieve registered clients: %s", cpp_strerror(r).c_str()); @@ -137,7 +137,7 @@ int expire_tags(cls_method_context_t hctx, const std::string *skip_client_id) { if (!vals.empty()) { last_read = vals.rbegin()->first; } - } while (r == MAX_KEYS_READ); + } while (more); // cannot expire tags if a client hasn't committed yet if (minimum_tag_tid == std::numeric_limits::max()) { @@ -153,8 +153,8 @@ int expire_tags(cls_method_context_t hctx, const std::string *skip_client_id) { last_read = HEADER_KEY_TAG_PREFIX; do { std::map vals; - r = cls_cxx_map_get_vals(hctx, last_read, HEADER_KEY_TAG_PREFIX, - MAX_KEYS_READ, &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, HEADER_KEY_TAG_PREFIX, + MAX_KEYS_READ, &vals, &more); if (r < 0 && r != -ENOENT) { CLS_ERR("failed to retrieve tags: %s", cpp_strerror(r).c_str()); return r; @@ -192,7 +192,7 @@ int expire_tags(cls_method_context_t hctx, const std::string *skip_client_id) { } } - if (tag_pass != TAG_PASS_DONE && vals.size() < MAX_KEYS_READ) { + if (tag_pass != TAG_PASS_DONE && !more) { last_read = HEADER_KEY_TAG_PREFIX; ++tag_pass; } else if (!vals.empty()) { @@ -211,8 +211,9 @@ int get_client_list_range(cls_method_context_t hctx, } std::map vals; + bool more; int r = cls_cxx_map_get_vals(hctx, last_read, HEADER_KEY_CLIENT_PREFIX, - max_return, &vals); + max_return, &vals, &more); if (r < 0) { CLS_ERR("failed to retrieve omap values: %s", cpp_strerror(r).c_str()); return r; @@ -1022,8 +1023,9 @@ int journal_tag_list(cls_method_context_t hctx, bufferlist *in, std::string last_read = HEADER_KEY_TAG_PREFIX; do { std::map vals; + bool more; r = cls_cxx_map_get_vals(hctx, last_read, HEADER_KEY_TAG_PREFIX, - MAX_KEYS_READ, &vals); + MAX_KEYS_READ, &vals, &more); if (r < 0 && r != -ENOENT) { CLS_ERR("failed to retrieve tags: %s", cpp_strerror(r).c_str()); return r; @@ -1062,7 +1064,7 @@ int journal_tag_list(cls_method_context_t hctx, bufferlist *in, } } - if (tag_pass != TAG_PASS_DONE && vals.size() < MAX_KEYS_READ) { + if (tag_pass != TAG_PASS_DONE && !more) { last_read = HEADER_KEY_TAG_PREFIX; ++tag_pass; } else if (!vals.empty()) { diff --git a/src/cls/log/cls_log.cc b/src/cls/log/cls_log.cc index bfc0e499f0228..411cbc91954d4 100644 --- a/src/cls/log/cls_log.cc +++ b/src/cls/log/cls_log.cc @@ -170,24 +170,22 @@ static int cls_log_list(cls_method_context_t hctx, bufferlist *in, bufferlist *o if (!max_entries || max_entries > MAX_ENTRIES) max_entries = MAX_ENTRIES; - int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries + 1, &keys); + cls_log_list_ret ret; + + int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys, &ret.truncated); if (rc < 0) return rc; - cls_log_list_ret ret; - list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - size_t i; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; marker = index; if (use_time_boundary && index.compare(0, to_index.size(), to_index) >= 0) { - done = true; + ret.truncated = false; break; } @@ -202,11 +200,9 @@ static int cls_log_list(cls_method_context_t hctx, bufferlist *in, bufferlist *o } } - if (iter == keys.end()) - done = true; - - ret.marker = marker; - ret.truncated = !done; + if (ret.truncated) { + ret.marker = marker; + } ::encode(ret, *out); @@ -244,16 +240,16 @@ static int cls_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlist *o #define MAX_TRIM_ENTRIES 1000 size_t max_entries = MAX_TRIM_ENTRIES; + bool more; - int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys); + int rc = cls_cxx_map_get_vals(hctx, from_index, log_index_prefix, max_entries, &keys, &more); if (rc < 0) return rc; map::iterator iter = keys.begin(); - size_t i; bool removed = false; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; CLS_LOG(20, "index=%s to_index=%s", index.c_str(), to_index.c_str()); diff --git a/src/cls/lua/cls_lua.cc b/src/cls/lua/cls_lua.cc index 1e15b4ed6fe11..5cb5bc80ebb3e 100644 --- a/src/cls/lua/cls_lua.cc +++ b/src/cls/lua/cls_lua.cc @@ -429,7 +429,8 @@ static int clslua_map_get_keys(lua_State *L) int max_to_get = luaL_checkinteger(L, 2); std::set keys; - int ret = cls_cxx_map_get_keys(hctx, start_after, max_to_get, &keys); + bool more; + int ret = cls_cxx_map_get_keys(hctx, start_after, max_to_get, &keys, &more); if (ret < 0) return clslua_opresult(L, 0, ret, 0); @@ -456,8 +457,9 @@ static int clslua_map_get_vals(lua_State *L) int max_to_get = luaL_checkinteger(L, 3); map kvpairs; + bool more; int ret = cls_cxx_map_get_vals(hctx, start_after, filter_prefix, - max_to_get, &kvpairs); + max_to_get, &kvpairs, &more); if (ret < 0) return clslua_opresult(L, 0, ret, 0); diff --git a/src/cls/rbd/cls_rbd.cc b/src/cls/rbd/cls_rbd.cc index 53a0d4efae976..2751006733882 100644 --- a/src/cls/rbd/cls_rbd.cc +++ b/src/cls/rbd/cls_rbd.cc @@ -1087,10 +1087,11 @@ int remove_parent(cls_method_context_t hctx, bufferlist *in, bufferlist *out) int max_read = RBD_MAX_KEYS_READ; vector snap_ids; string last_read = RBD_SNAP_KEY_PREFIX; + bool more; do { set keys; - r = cls_cxx_map_get_keys(hctx, last_read, max_read, &keys); + r = cls_cxx_map_get_keys(hctx, last_read, max_read, &keys, &more); if (r < 0) { return r; } @@ -1125,7 +1126,7 @@ int remove_parent(cls_method_context_t hctx, bufferlist *in, bufferlist *out) if (!keys.empty()) { last_read = *(keys.rbegin()); } - } while (r == max_read); + } while (more); } cls_rbd_parent parent; @@ -1370,10 +1371,11 @@ int get_snapcontext(cls_method_context_t hctx, bufferlist *in, bufferlist *out) int max_read = RBD_MAX_KEYS_READ; vector snap_ids; string last_read = RBD_SNAP_KEY_PREFIX; + bool more; do { set keys; - r = cls_cxx_map_get_keys(hctx, last_read, max_read, &keys); + r = cls_cxx_map_get_keys(hctx, last_read, max_read, &keys, &more); if (r < 0) return r; @@ -1386,7 +1388,7 @@ int get_snapcontext(cls_method_context_t hctx, bufferlist *in, bufferlist *out) } if (!keys.empty()) last_read = *(keys.rbegin()); - } while (r == max_read); + } while (more); uint64_t snap_seq; r = read_key(hctx, "snap_seq", &snap_seq); @@ -1631,10 +1633,11 @@ int snapshot_add(cls_method_context_t hctx, bufferlist *in, bufferlist *out) int max_read = RBD_MAX_KEYS_READ; uint64_t total_read = 0; string last_read = RBD_SNAP_KEY_PREFIX; + bool more; do { map vals; r = cls_cxx_map_get_vals(hctx, last_read, RBD_SNAP_KEY_PREFIX, - max_read, &vals); + max_read, &vals, &more); if (r < 0) return r; @@ -1668,7 +1671,7 @@ int snapshot_add(cls_method_context_t hctx, bufferlist *in, bufferlist *out) if (!vals.empty()) last_read = vals.rbegin()->first; - } while (r == RBD_MAX_KEYS_READ); + } while (more); // snapshot inherits parent, if any cls_rbd_parent parent; @@ -1729,10 +1732,11 @@ int snapshot_rename(cls_method_context_t hctx, bufferlist *in, bufferlist *out) int max_read = RBD_MAX_KEYS_READ; string last_read = RBD_SNAP_KEY_PREFIX; + bool more; do { map vals; r = cls_cxx_map_get_vals(hctx, last_read, RBD_SNAP_KEY_PREFIX, - max_read, &vals); + max_read, &vals, &more); if (r < 0) return r; @@ -1754,7 +1758,7 @@ int snapshot_rename(cls_method_context_t hctx, bufferlist *in, bufferlist *out) } if (!vals.empty()) last_read = vals.rbegin()->first; - } while (r == RBD_MAX_KEYS_READ); + } while (more); key_from_snap_id(src_snap_id, &src_snap_key); r = read_key(hctx, src_snap_key, &snap_meta); @@ -2164,15 +2168,15 @@ int dir_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) } int max_read = RBD_MAX_KEYS_READ; - int r = max_read; map images; string last_read = dir_key_for_name(start_after); + bool more = true; - while (r == max_read && images.size() < max_return) { + while (more && images.size() < max_return) { map vals; CLS_LOG(20, "last_read = '%s'", last_read.c_str()); - r = cls_cxx_map_get_vals(hctx, last_read, RBD_DIR_NAME_KEY_PREFIX, - max_read, &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, RBD_DIR_NAME_KEY_PREFIX, + max_read, &vals, &more); if (r < 0) { CLS_ERR("error reading directory by name: %s", cpp_strerror(r).c_str()); return r; @@ -2648,11 +2652,12 @@ int metadata_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) map data; string last_read = metadata_key_for_name(start_after); int max_read = max_return ? MIN(RBD_MAX_KEYS_READ, max_return) : RBD_MAX_KEYS_READ; + bool more; do { map raw_data; int r = cls_cxx_map_get_vals(hctx, last_read, RBD_METADATA_KEY_PREFIX, - max_read, &raw_data); + max_read, &raw_data, &more); if (r < 0) { CLS_ERR("failed to read the vals off of disk: %s", cpp_strerror(r).c_str()); return r; @@ -2664,13 +2669,13 @@ int metadata_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) for (; it != raw_data.end(); ++it) data[metadata_name_from_key(it->first)].swap(it->second); - if (r < max_read) + if (!more) break; last_read = raw_data.rbegin()->first; if (max_return) max_read = MIN(RBD_MAX_KEYS_READ, max_return - data.size()); - } while (max_read); + } while (more); ::encode(data, *out); return 0; @@ -3166,11 +3171,11 @@ int read_peers(cls_method_context_t hctx, std::vector *peers) { std::string last_read = PEER_KEY_PREFIX; int max_read = RBD_MAX_KEYS_READ; - int r = max_read; - while (r == max_read) { + bool more = true; + while (more) { std::map vals; - r = cls_cxx_map_get_vals(hctx, last_read, PEER_KEY_PREFIX.c_str(), - max_read, &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, PEER_KEY_PREFIX.c_str(), + max_read, &vals, &more); if (r < 0) { CLS_ERR("error reading peers: %s", cpp_strerror(r).c_str()); return r; @@ -3462,13 +3467,13 @@ int image_status_list(cls_method_context_t hctx, map *mirror_statuses) { std::string last_read = image_key(start_after); int max_read = RBD_MAX_KEYS_READ; - int r = max_read; + bool more = true; - while (r == max_read && mirror_images->size() < max_return) { + while (more && mirror_images->size() < max_return) { std::map vals; CLS_LOG(20, "last_read = '%s'", last_read.c_str()); - r = cls_cxx_map_get_vals(hctx, last_read, IMAGE_KEY_PREFIX, max_read, - &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, IMAGE_KEY_PREFIX, max_read, + &vals, &more); if (r < 0) { CLS_ERR("error reading mirror image directory by name: %s", cpp_strerror(r).c_str()); @@ -3526,11 +3531,11 @@ int image_status_get_summary(cls_method_context_t hctx, string last_read = IMAGE_KEY_PREFIX; int max_read = RBD_MAX_KEYS_READ; - r = max_read; - while (r == max_read) { + bool more = true; + while (more) { map vals; r = cls_cxx_map_get_vals(hctx, last_read, IMAGE_KEY_PREFIX, - max_read, &vals); + max_read, &vals, &more); if (r < 0) { CLS_ERR("error reading mirrored images: %s", cpp_strerror(r).c_str()); return r; @@ -3586,11 +3591,11 @@ int image_status_remove_down(cls_method_context_t hctx) { string last_read = STATUS_GLOBAL_KEY_PREFIX; int max_read = RBD_MAX_KEYS_READ; - r = max_read; - while (r == max_read) { + bool more = true; + while (more) { map vals; r = cls_cxx_map_get_vals(hctx, last_read, STATUS_GLOBAL_KEY_PREFIX, - max_read, &vals); + max_read, &vals, &more); if (r < 0) { CLS_ERR("error reading mirrored images: %s", cpp_strerror(r).c_str()); return r; @@ -3638,11 +3643,11 @@ int instances_list(cls_method_context_t hctx, std::vector *instance_ids) { std::string last_read = INSTANCE_KEY_PREFIX; int max_read = RBD_MAX_KEYS_READ; - int r = max_read; - while (r == max_read) { + bool more = true; + while (more) { std::map vals; - r = cls_cxx_map_get_vals(hctx, last_read, INSTANCE_KEY_PREFIX.c_str(), - max_read, &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, INSTANCE_KEY_PREFIX.c_str(), + max_read, &vals, &more); if (r < 0) { if (r != -ENOENT) { CLS_ERR("error reading mirror instances: %s", cpp_strerror(r).c_str()); @@ -4049,15 +4054,15 @@ int mirror_image_list(cls_method_context_t hctx, bufferlist *in, } int max_read = RBD_MAX_KEYS_READ; - int r = max_read; + bool more = true; std::map mirror_images; std::string last_read = mirror::image_key(start_after); - while (r == max_read && mirror_images.size() < max_return) { + while (more && mirror_images.size() < max_return) { std::map vals; CLS_LOG(20, "last_read = '%s'", last_read.c_str()); - r = cls_cxx_map_get_vals(hctx, last_read, mirror::IMAGE_KEY_PREFIX, - max_read, &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, mirror::IMAGE_KEY_PREFIX, + max_read, &vals, &more); if (r < 0) { CLS_ERR("error reading mirror image directory by name: %s", cpp_strerror(r).c_str()); @@ -4469,15 +4474,15 @@ int group_dir_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) } int max_read = RBD_MAX_KEYS_READ; - int r = max_read; + bool more = true; map groups; string last_read = dir_key_for_name(start_after); - while (r == max_read && groups.size() < max_return) { + while (more && groups.size() < max_return) { map vals; CLS_LOG(20, "last_read = '%s'", last_read.c_str()); - r = cls_cxx_map_get_vals(hctx, last_read, RBD_DIR_NAME_KEY_PREFIX, - max_read, &vals); + int r = cls_cxx_map_get_vals(hctx, last_read, RBD_DIR_NAME_KEY_PREFIX, + max_read, &vals, &more); if (r < 0) { CLS_ERR("error reading directory by name: %s", cpp_strerror(r).c_str()); return r; @@ -4723,12 +4728,12 @@ int group_image_list(cls_method_context_t hctx, std::map vals; string last_read = start_after.image_key(); std::vector res; - int keys_read; + bool more; do { - keys_read = cls_cxx_map_get_vals(hctx, last_read,cls::rbd::RBD_GROUP_IMAGE_KEY_PREFIX, - max_read, &vals); - if (keys_read < 0) - return keys_read; + int r = cls_cxx_map_get_vals(hctx, last_read,cls::rbd::RBD_GROUP_IMAGE_KEY_PREFIX, + max_read, &vals, &more); + if (r < 0) + return r; for (map::iterator it = vals.begin(); it != vals.end() && res.size() < max_return; ++it) { @@ -4755,7 +4760,7 @@ int group_image_list(cls_method_context_t hctx, last_read = res.rbegin()->spec.image_key(); } - } while ((keys_read == RBD_MAX_KEYS_READ) && (res.size() < max_return)); + } while (more && (res.size() < max_return)); ::encode(res, *out); return 0; @@ -5041,6 +5046,8 @@ int trash_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) map data; string last_read = trash::image_key(start_after); + int max_read = RBD_MAX_KEYS_READ; + bool more = true; CLS_LOG(20, "trash_get_images"); while (data.size() < max_return) { @@ -5048,7 +5055,7 @@ int trash_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) int max_read = std::min(RBD_MAX_KEYS_READ, max_return - data.size()); int r = cls_cxx_map_get_vals(hctx, last_read, trash::IMAGE_KEY_PREFIX, - max_read, &raw_data); + max_read, &raw_data, &more); if (r < 0) { CLS_ERR("failed to read the vals off of disk: %s", cpp_strerror(r).c_str()); @@ -5063,7 +5070,7 @@ int trash_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) ::decode(data[trash::image_id_from_key(it->first)], it->second); } - if (r < max_read) { + if (!more) { break; } diff --git a/src/cls/rgw/cls_rgw.cc b/src/cls/rgw/cls_rgw.cc index 9af96fe6ab485..f7e2f6b656847 100644 --- a/src/cls/rgw/cls_rgw.cc +++ b/src/cls/rgw/cls_rgw.cc @@ -145,9 +145,9 @@ static int log_index_operation(cls_method_context_t hctx, cls_rgw_obj_key& obj_k * read list of objects, skips objects in the ugly namespace */ static int get_obj_vals(cls_method_context_t hctx, const string& start, const string& filter_prefix, - int num_entries, map *pkeys) + int num_entries, map *pkeys, bool *pmore) { - int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys); + int ret = cls_cxx_map_get_vals(hctx, start, filter_prefix, num_entries, pkeys, pmore); if (ret < 0) return ret; @@ -183,7 +183,7 @@ static int get_obj_vals(cls_method_context_t hctx, const string& start, const st string new_start = c; /* now get some more keys */ - ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys); + ret = cls_cxx_map_get_vals(hctx, new_start, filter_prefix, num_entries - pkeys->size(), &new_keys, pmore); if (ret < 0) return ret; @@ -405,10 +405,11 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) string start_key; encode_list_index_key(hctx, op.start_obj, &start_key); bool done = false; - uint32_t left_to_read = op.num_entries + 1; + uint32_t left_to_read = op.num_entries; + bool more; do { - rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys); + rc = get_obj_vals(hctx, start_key, op.filter_prefix, left_to_read, &keys, &more); if (rc < 0) return rc; @@ -458,8 +459,7 @@ int rgw_bucket_list(cls_method_context_t hctx, bufferlist *in, bufferlist *out) } } while (left_to_read > 0 && !done); - ret.is_truncated = (left_to_read == 0) && /* we found more entries than we were requested, meaning response is truncated */ - !done; + ret.is_truncated = more && !done; ::encode(ret, *out); return 0; @@ -482,9 +482,10 @@ static int check_index(cls_method_context_t hctx, struct rgw_bucket_dir_header * #define CHECK_CHUNK_SIZE 1000 bool done = false; + bool more; do { - rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys); + rc = get_obj_vals(hctx, start_obj, filter_prefix, CHECK_CHUNK_SIZE, &keys, &more); if (rc < 0) return rc; @@ -1157,9 +1158,10 @@ class BIVerObjEntry { get_list_index_key(instance_entry, &list_idx); /* this is the current head, need to update! */ map keys; + bool more; string filter = key.name; /* list key starts with key name, filter it to avoid a case where we cross to different namespace */ - int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys); + int ret = cls_cxx_map_get_vals(hctx, list_idx, filter, 1, &keys, &more); if (ret < 0) { return ret; } @@ -2260,7 +2262,7 @@ static int rgw_bi_put_op(cls_method_context_t hctx, bufferlist *in, bufferlist * } static int list_plain_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { string filter = name; string start_key = marker; @@ -2270,59 +2272,52 @@ static int list_plain_entries(cls_method_context_t hctx, const string& name, con int count = 0; map keys; - do { - if (count >= (int)max) { + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max, &keys, pmore); + if (ret < 0) { + return ret; + } + + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + if (iter->first >= end_key) { + /* past the end of plain namespace */ return count; } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_GET_NUM_KEYS, &keys); - if (ret < 0) { - return ret; - } - - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - if (iter->first >= end_key) { - /* past the end of plain namespace */ - return count; - } - rgw_cls_bi_entry entry; - entry.type = PlainIdx; - entry.idx = iter->first; - entry.data = iter->second; + rgw_cls_bi_entry entry; + entry.type = PlainIdx; + entry.idx = iter->first; + entry.data = iter->second; - bufferlist::iterator biter = entry.data.begin(); + bufferlist::iterator biter = entry.data.begin(); - rgw_bucket_dir_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__); - return -EIO; - } + rgw_bucket_dir_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer", __func__); + return -EIO; + } - CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); + CLS_LOG(20, "%s(): entry.idx=%s e.key.name=%s", __func__, escape_str(entry.idx).c_str(), escape_str(e.key.name).c_str()); - if (!name.empty() && e.key.name != name) { - return count; - } + if (!name.empty() && e.key.name != name) { + return count; + } - entries->push_back(entry); - count++; - if (count >= (int)max) { - return count; - } - start_key = entry.idx; + entries->push_back(entry); + count++; + if (count >= (int)max) { + return count; } - } while (!keys.empty()); + start_key = entry.idx; + } return count; } static int list_instance_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { cls_rgw_obj_key key(name); string first_instance_idx; @@ -2341,66 +2336,63 @@ static int list_instance_entries(cls_method_context_t hctx, const string& name, } int count = 0; map keys; - bool started = true; - do { - if (count >= (int)max) { - return count; - } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret; - if (started) { - ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]); - if (ret == -ENOENT) { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } - started = false; - } else { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } + bufferlist k; + int ret = cls_cxx_map_get_val(hctx, start_key, &k); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + bool found_first = (ret == 0); + if (found_first) { + --max; + } + if (max > 0) { + ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore); CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size()); if (ret < 0) { return ret; } + } + if (found_first) { + keys[start_key].claim(k); + } - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - rgw_cls_bi_entry entry; - entry.type = InstanceIdx; - entry.idx = iter->first; - entry.data = iter->second; - - if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { - return count; - } + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + rgw_cls_bi_entry entry; + entry.type = InstanceIdx; + entry.idx = iter->first; + entry.data = iter->second; - CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); + if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + return count; + } - bufferlist::iterator biter = entry.data.begin(); + CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); - rgw_bucket_dir_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); - return -EIO; - } + bufferlist::iterator biter = entry.data.begin(); - if (!name.empty() && e.key.name != name) { - return count; - } + rgw_bucket_dir_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); + return -EIO; + } - entries->push_back(entry); - count++; - start_key = entry.idx; + if (!name.empty() && e.key.name != name) { + return count; } - } while (!keys.empty()); + + entries->push_back(entry); + count++; + start_key = entry.idx; + } return count; } static int list_olh_entries(cls_method_context_t hctx, const string& name, const string& marker, uint32_t max, - list *entries) + list *entries, bool *pmore) { cls_rgw_obj_key key(name); string first_instance_idx; @@ -2419,60 +2411,59 @@ static int list_olh_entries(cls_method_context_t hctx, const string& name, const } int count = 0; map keys; - bool started = true; - do { - if (count >= (int)max) { - return count; - } - keys.clear(); -#define BI_GET_NUM_KEYS 128 - int ret; - if (started) { - ret = cls_cxx_map_get_val(hctx, start_key, &keys[start_key]); - if (ret == -ENOENT) { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } - started = false; - } else { - ret = cls_cxx_map_get_vals(hctx, start_key, string(), BI_GET_NUM_KEYS, &keys); - } + int ret; + bufferlist k; + ret = cls_cxx_map_get_val(hctx, start_key, &k); + if (ret < 0 && ret != -ENOENT) { + return ret; + } + bool found_first = (ret == 0); + if (found_first) { + --max; + } + if (max > 0) { + ret = cls_cxx_map_get_vals(hctx, start_key, string(), max, &keys, pmore); CLS_LOG(20, "%s(): start_key=%s first_instance_idx=%s keys.size()=%d", __func__, escape_str(start_key).c_str(), escape_str(first_instance_idx).c_str(), (int)keys.size()); if (ret < 0) { return ret; } + } - map::iterator iter; - for (iter = keys.begin(); iter != keys.end(); ++iter) { - rgw_cls_bi_entry entry; - entry.type = OLHIdx; - entry.idx = iter->first; - entry.data = iter->second; + if (found_first) { + keys[start_key].claim(k); + } - if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { - return count; - } + map::iterator iter; + for (iter = keys.begin(); iter != keys.end(); ++iter) { + rgw_cls_bi_entry entry; + entry.type = OLHIdx; + entry.idx = iter->first; + entry.data = iter->second; - CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); + if (!filter.empty() && entry.idx.compare(0, filter.size(), filter) != 0) { + return count; + } - bufferlist::iterator biter = entry.data.begin(); + CLS_LOG(20, "%s(): entry.idx=%s", __func__, escape_str(entry.idx).c_str()); - rgw_bucket_olh_entry e; - try { - ::decode(e, biter); - } catch (buffer::error& err) { - CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); - return -EIO; - } + bufferlist::iterator biter = entry.data.begin(); - if (!name.empty() && e.key.name != name) { - return count; - } + rgw_bucket_olh_entry e; + try { + ::decode(e, biter); + } catch (buffer::error& err) { + CLS_LOG(0, "ERROR: %s(): failed to decode buffer (size=%d)", __func__, entry.data.length()); + return -EIO; + } - entries->push_back(entry); - count++; - start_key = entry.idx; + if (!name.empty() && e.key.name != name) { + return count; } - } while (!keys.empty()); + + entries->push_back(entry); + count++; + start_key = entry.idx; + } return count; } @@ -2493,9 +2484,10 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist string filter = op.name; #define MAX_BI_LIST_ENTRIES 1000 - int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES) + 1; /* one extra entry for identifying truncation */ + int32_t max = (op.max < MAX_BI_LIST_ENTRIES ? op.max : MAX_BI_LIST_ENTRIES); string start_key = op.marker; - int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries); + bool more; + int ret = list_plain_entries(hctx, op.name, op.marker, max, &op_ret.entries, &more); if (ret < 0) { CLS_LOG(0, "ERROR: %s(): list_plain_entries retured ret=%d", __func__, ret); return ret; @@ -2504,23 +2496,27 @@ static int rgw_bi_list_op(cls_method_context_t hctx, bufferlist *in, bufferlist CLS_LOG(20, "found %d plain entries", count); - ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries); - if (ret < 0) { - CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret); - return ret; + if (!more) { + ret = list_instance_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more); + if (ret < 0) { + CLS_LOG(0, "ERROR: %s(): list_instance_entries retured ret=%d", __func__, ret); + return ret; + } + + count += ret; } - count += ret; + if (!more) { + ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries, &more); + if (ret < 0) { + CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret); + return ret; + } - ret = list_olh_entries(hctx, op.name, op.marker, max - count, &op_ret.entries); - if (ret < 0) { - CLS_LOG(0, "ERROR: %s(): list_olh_entries retured ret=%d", __func__, ret); - return ret; + count += ret; } - count += ret; - - op_ret.is_truncated = (count >= max); + op_ret.is_truncated = (count >= max) || more; while (count >= max) { op_ret.entries.pop_back(); count--; @@ -2582,45 +2578,40 @@ static int bi_log_iterate_entries(cls_method_context_t hctx, const string& marke string filter; - do { -#define BI_NUM_KEYS 128 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, BI_NUM_KEYS, &keys); - if (ret < 0) - return ret; + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - rgw_bi_log_entry e; + uint32_t num_keys = keys.size(); - CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length()); + for (; iter != keys.end(); ++iter,++i) { + const string& key = iter->first; + rgw_bi_log_entry e; - if (key.compare(end_key) > 0) - return 0; + CLS_LOG(0, "bi_log_iterate_entries key=%s bl.length=%d\n", key.c_str(), (int)iter->second.length()); - ret = bi_log_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (key.compare(end_key) > 0) { + key_iter = key; + return 0; + } - if (max_entries && (i >= max_entries)) { - if (truncated) - *truncated = true; - key_iter = key; - return 0; - } + ret = bi_log_record_decode(iter->second, e); + if (ret < 0) + return ret; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; - i++; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; + if (i == num_keys - 1) { + key_iter = key; } - --iter; - start_key = iter->first; - } while (true); + } + return 0; } @@ -2862,58 +2853,53 @@ static int usage_iterate_range(cls_method_context_t hctx, uint64_t start, uint64 start_key = key_iter; } - do { - CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str()); - int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, NUM_KEYS, &keys); - if (ret < 0) - return ret; + CLS_LOG(20, "usage_iterate_range start_key=%s", start_key.c_str()); + int ret = cls_cxx_map_get_vals(hctx, start_key, filter_prefix, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - rgw_usage_log_entry e; + uint32_t num_keys = keys.size(); - if (!by_user && key.compare(end_key) >= 0) { - CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); - return 0; - } + for (; iter != keys.end(); ++iter,++i) { + const string& key = iter->first; + rgw_usage_log_entry e; - if (by_user && key.compare(0, user_key.size(), user_key) != 0) { - CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); - return 0; - } + if (!by_user && key.compare(end_key) >= 0) { + CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + return 0; + } - ret = usage_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (by_user && key.compare(0, user_key.size(), user_key) != 0) { + CLS_LOG(20, "usage_iterate_range reached key=%s, done", key.c_str()); + return 0; + } - if (e.epoch < start) - continue; + ret = usage_record_decode(iter->second, e); + if (ret < 0) + return ret; - /* keys are sorted by epoch, so once we're past end we're done */ - if (e.epoch >= end) - return 0; + if (e.epoch < start) + continue; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; + /* keys are sorted by epoch, so once we're past end we're done */ + if (e.epoch >= end) + return 0; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; - i++; - if (max_entries && (i > max_entries)) { - CLS_LOG(20, "usage_iterate_range reached max_entries (%d), done", max_entries); - *truncated = true; - key_iter = key; - return 0; - } + + if (i == num_keys - 1) { + key_iter = key; + return 0; } - --iter; - start_key = iter->first; - } while (true); + } return 0; } @@ -2999,7 +2985,9 @@ int rgw_user_usage_log_trim(cls_method_context_t hctx, bufferlist *in, bufferlis } string iter; - ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, 0, NULL, usage_log_trim_cb, NULL); + bool more; +#define MAX_USAGE_TRIM_ENTRIES 128 + ret = usage_iterate_range(hctx, op.start_epoch, op.end_epoch, op.user, iter, MAX_USAGE_TRIM_ENTRIES, &more, usage_log_trim_cb, NULL); if (ret < 0) return ret; @@ -3197,50 +3185,42 @@ static int gc_iterate_entries(cls_method_context_t hctx, const string& marker, b string filter; - do { -#define GC_NUM_KEYS 32 - int ret = cls_cxx_map_get_vals(hctx, start_key, filter, GC_NUM_KEYS, &keys); - if (ret < 0) - return ret; + int ret = cls_cxx_map_get_vals(hctx, start_key, filter, max_entries, &keys, truncated); + if (ret < 0) + return ret; - map::iterator iter = keys.begin(); - if (iter == keys.end()) - break; + map::iterator iter = keys.begin(); + if (iter == keys.end()) + return 0; - for (; iter != keys.end(); ++iter) { - const string& key = iter->first; - cls_rgw_gc_obj_info e; + uint32_t num_keys = keys.size(); - CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str()); + for (; iter != keys.end(); ++iter, ++i) { + const string& key = iter->first; + cls_rgw_gc_obj_info e; - if (!end_key.empty() && key.compare(end_key) >= 0) - return 0; + CLS_LOG(10, "gc_iterate_entries key=%s\n", key.c_str()); - if (!key_in_index(key, GC_OBJ_TIME_INDEX)) - return 0; + if (!end_key.empty() && key.compare(end_key) >= 0) + return 0; - ret = gc_record_decode(iter->second, e); - if (ret < 0) - return ret; + if (!key_in_index(key, GC_OBJ_TIME_INDEX)) + return 0; - if (max_entries && (i >= max_entries)) { - if (truncated) - *truncated = true; - --iter; - key_iter = iter->first; - return 0; - } + ret = gc_record_decode(iter->second, e); + if (ret < 0) + return ret; - ret = cb(hctx, key, e, param); - if (ret < 0) - return ret; - i++; + ret = cb(hctx, key, e, param); + if (ret < 0) + return ret; + if (i == num_keys - 1) { + key_iter = key; } - --iter; - start_key = iter->first; - } while (true); + } + return 0; } @@ -3274,7 +3254,8 @@ static int rgw_cls_gc_list(cls_method_context_t hctx, bufferlist *in, bufferlist } cls_rgw_gc_list_ret op_ret; - int ret = gc_list_entries(hctx, op.marker, op.max, op.expired_only, +#define GC_LIST_ENTRIES_DEFAULT 128 + int ret = gc_list_entries(hctx, op.marker, (op.max ? op.max : GC_LIST_ENTRIES_DEFAULT), op.expired_only, op_ret.entries, &op_ret.truncated, op_ret.next_marker); if (ret < 0) return ret; @@ -3384,7 +3365,8 @@ static int rgw_cls_lc_get_next_entry(cls_method_context_t hctx, bufferlist *in, map vals; string filter_prefix; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals); + bool more; + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, 1, &vals, &more); if (ret < 0) return ret; map::iterator it; @@ -3419,7 +3401,7 @@ static int rgw_cls_lc_list_entries(cls_method_context_t hctx, bufferlist *in, bu bufferlist::iterator iter; map vals; string filter_prefix; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals); + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, op.max_entries, &vals, &op_ret.is_truncated); if (ret < 0) return ret; map::iterator it; @@ -3524,8 +3506,8 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis string filter_prefix; #define MAX_RESHARD_LIST_ENTRIES 1000 /* one extra entry for identifying truncation */ - int32_t max = (op.max < MAX_RESHARD_LIST_ENTRIES ? op.max : MAX_RESHARD_LIST_ENTRIES) + 1; - int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals); + int32_t max = (op.max && (op.max < MAX_RESHARD_LIST_ENTRIES) ? op.max : MAX_RESHARD_LIST_ENTRIES); + int ret = cls_cxx_map_get_vals(hctx, op.marker, filter_prefix, max, &vals, &op_ret.is_truncated); if (ret < 0) return ret; map::iterator it; @@ -3541,7 +3523,6 @@ static int rgw_reshard_list(cls_method_context_t hctx, bufferlist *in, bufferlis } op_ret.entries.push_back(entry); } - op_ret.is_truncated = op.max && (vals.size() > op.max); ::encode(op_ret, *out); return 0; } diff --git a/src/cls/rgw/cls_rgw_ops.h b/src/cls/rgw/cls_rgw_ops.h index 538443adf40b4..07ebeea12fde7 100644 --- a/src/cls/rgw/cls_rgw_ops.h +++ b/src/cls/rgw/cls_rgw_ops.h @@ -1123,18 +1123,23 @@ WRITE_CLASS_ENCODER(cls_rgw_lc_list_entries_op) struct cls_rgw_lc_list_entries_ret { map entries; + bool is_truncated{false}; cls_rgw_lc_list_entries_ret() {} void encode(bufferlist& bl) const { - ENCODE_START(1, 1, bl); + ENCODE_START(2, 1, bl); ::encode(entries, bl); + ::encode(is_truncated, bl); ENCODE_FINISH(bl); } void decode(bufferlist::iterator& bl) { - DECODE_START(1, bl); + DECODE_START(2, bl); ::decode(entries, bl); + if (struct_v >= 2) { + ::decode(is_truncated, bl); + } DECODE_FINISH(bl); } diff --git a/src/cls/statelog/cls_statelog.cc b/src/cls/statelog/cls_statelog.cc index f404fa77b60b3..b2a13764d1e8f 100644 --- a/src/cls/statelog/cls_statelog.cc +++ b/src/cls/statelog/cls_statelog.cc @@ -168,21 +168,20 @@ static int cls_statelog_list(cls_method_context_t hctx, bufferlist *in, bufferli if (!max_entries || max_entries > MAX_ENTRIES) max_entries = MAX_ENTRIES; - int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries + 1, &keys); + cls_statelog_list_ret ret; + + int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries, &keys, &ret.truncated); if (rc < 0) return rc; CLS_LOG(20, "from_index=%s match_prefix=%s", from_index.c_str(), match_prefix.c_str()); - cls_statelog_list_ret ret; list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - size_t i; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; marker = index; @@ -197,12 +196,9 @@ static int cls_statelog_list(cls_method_context_t hctx, bufferlist *in, bufferli } } - if (iter == keys.end()) - done = true; - else + if (ret.truncated) { ret.marker = marker; - - ret.truncated = !done; + } ::encode(ret, *out); diff --git a/src/cls/timeindex/cls_timeindex.cc b/src/cls/timeindex/cls_timeindex.cc index 202b653567ea7..bed96e5cd74d6 100644 --- a/src/cls/timeindex/cls_timeindex.cc +++ b/src/cls/timeindex/cls_timeindex.cc @@ -120,21 +120,20 @@ static int cls_timeindex_list(cls_method_context_t hctx, max_entries = MAX_LIST_ENTRIES; } + cls_timeindex_list_ret ret; + int rc = cls_cxx_map_get_vals(hctx, from_index, TIMEINDEX_PREFIX, - max_entries + 1, &keys); + max_entries, &keys, &ret.truncated); if (rc < 0) { return rc; } - cls_timeindex_list_ret ret; - list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - for (size_t i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; bufferlist& bl = iter->second; @@ -142,7 +141,7 @@ static int cls_timeindex_list(cls_method_context_t hctx, if (use_time_boundary && index.compare(0, to_index.size(), to_index) >= 0) { CLS_LOG(20, "DEBUG: cls_timeindex_list: finishing on to_index=%s", to_index.c_str()); - done = true; + ret.truncated = false; break; } @@ -159,12 +158,7 @@ static int cls_timeindex_list(cls_method_context_t hctx, } } - if (iter == keys.end()) { - done = true; - } - ret.marker = marker; - ret.truncated = !done; ::encode(ret, *out); @@ -203,8 +197,10 @@ static int cls_timeindex_trim(cls_method_context_t hctx, to_index = op.to_marker; } + bool more; + int rc = cls_cxx_map_get_vals(hctx, from_index, TIMEINDEX_PREFIX, - MAX_TRIM_ENTRIES, &keys); + MAX_TRIM_ENTRIES, &keys, &more); if (rc < 0) { return rc; } @@ -212,7 +208,7 @@ static int cls_timeindex_trim(cls_method_context_t hctx, map::iterator iter = keys.begin(); bool removed = false; - for (size_t i = 0; i < MAX_TRIM_ENTRIES && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; CLS_LOG(20, "index=%s to_index=%s", index.c_str(), to_index.c_str()); diff --git a/src/cls/user/cls_user.cc b/src/cls/user/cls_user.cc index c616d1ae65639..7912578aa36c8 100644 --- a/src/cls/user/cls_user.cc +++ b/src/cls/user/cls_user.cc @@ -292,8 +292,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff max_entries = MAX_ENTRIES; string match_prefix; + cls_user_list_buckets_ret ret; - int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries + 1, &keys); + int rc = cls_cxx_map_get_vals(hctx, from_index, match_prefix, max_entries, &keys, &ret.truncated); if (rc < 0) return rc; @@ -301,21 +302,20 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff from_index.c_str(), to_index.c_str(), match_prefix.c_str()); - cls_user_list_buckets_ret ret; list& entries = ret.entries; map::iterator iter = keys.begin(); - bool done = false; string marker; - size_t i; - for (i = 0; i < max_entries && iter != keys.end(); ++i, ++iter) { + for (; iter != keys.end(); ++iter) { const string& index = iter->first; marker = index; - if (to_index_valid && to_index.compare(index) <= 0) + if (to_index_valid && to_index.compare(index) <= 0) { + ret.truncated = false; break; + } bufferlist& bl = iter->second; bufferlist::iterator biter = bl.begin(); @@ -328,12 +328,9 @@ static int cls_user_list_buckets(cls_method_context_t hctx, bufferlist *in, buff } } - if (iter == keys.end()) - done = true; - else + if (ret.truncated) { ret.marker = marker; - - ret.truncated = !done; + } ::encode(ret, *out); diff --git a/src/key_value_store/cls_kvs.cc b/src/key_value_store/cls_kvs.cc index 2a6f3ab67bdd1..183c8d6f9468b 100644 --- a/src/key_value_store/cls_kvs.cc +++ b/src/key_value_store/cls_kvs.cc @@ -30,7 +30,9 @@ static int get_idata_from_key(cls_method_context_t hctx, const string &key, int r = 0; std::map kvmap; - r = cls_cxx_map_get_vals(hctx, key_data(key).encoded(), "", 2, &kvmap); + bool more; + + r = cls_cxx_map_get_vals(hctx, key_data(key).encoded(), "", 2, &kvmap, &more); if (r < 0) { CLS_LOG(20, "error reading index for range %s: %d", key.c_str(), r); return r; @@ -99,7 +101,8 @@ static int get_next_idata(cls_method_context_t hctx, const index_data &idata, index_data &out_data) { int r = 0; std::map kvs; - r = cls_cxx_map_get_vals(hctx, idata.kdata.encoded(), "", 1, &kvs); + bool more; + r = cls_cxx_map_get_vals(hctx, idata.kdata.encoded(), "", 1, &kvs, &more); if (r < 0){ CLS_LOG(20, "getting kvs failed with error %d", r); return r; @@ -150,7 +153,8 @@ static int get_prev_idata(cls_method_context_t hctx, const index_data &idata, index_data &out_data) { int r = 0; std::map kvs; - r = cls_cxx_map_get_vals(hctx, "", "", LONG_MAX, &kvs); + bool more; + r = cls_cxx_map_get_vals(hctx, "", "", LONG_MAX, &kvs, &more); if (r < 0){ CLS_LOG(20, "getting kvs failed with error %d", r); return r; @@ -202,10 +206,11 @@ static int get_prev_idata_op(cls_method_context_t hctx, static int read_many(cls_method_context_t hctx, const set &keys, map * out) { int r = 0; + bool more; CLS_ERR("reading from a map of size %d, first key encoded is %s", (int)keys.size(), key_data(*keys.begin()).encoded().c_str()); r = cls_cxx_map_get_vals(hctx, key_data(*keys.begin()).encoded().c_str(), - "", LONG_MAX, out); + "", LONG_MAX, out, &more); if (r < 0) { CLS_ERR("getting omap vals failed with error %d", r); } @@ -599,7 +604,8 @@ static int maybe_read_for_balance(cls_method_context_t hctx, } //if the assert succeeded, it needs to be balanced - r = cls_cxx_map_get_vals(hctx, "", "", LONG_MAX, &odata.omap); + bool more; + r = cls_cxx_map_get_vals(hctx, "", "", LONG_MAX, &odata.omap, &more); if (r < 0){ CLS_LOG(20, "rebalance read: getting kvs failed with error %d", r); return r; diff --git a/src/objclass/class_api.cc b/src/objclass/class_api.cc index 724842c1cb134..cb8aa0677f8c2 100644 --- a/src/objclass/class_api.cc +++ b/src/objclass/class_api.cc @@ -405,7 +405,8 @@ int cls_cxx_snap_revert(cls_method_context_t hctx, snapid_t snapid) return (*pctx)->pg->do_osd_ops(*pctx, ops); } -int cls_cxx_map_get_all_vals(cls_method_context_t hctx, map* vals) +int cls_cxx_map_get_all_vals(cls_method_context_t hctx, map* vals, + bool *more) { PrimaryLogPG::OpContext **pctx = (PrimaryLogPG::OpContext **)hctx; vector ops(1); @@ -429,6 +430,7 @@ int cls_cxx_map_get_all_vals(cls_method_context_t hctx, map* bufferlist::iterator iter = op.outdata.begin(); try { ::decode(*vals, iter); + ::decode(*more, iter); } catch (buffer::error& err) { return -EIO; } @@ -436,7 +438,8 @@ int cls_cxx_map_get_all_vals(cls_method_context_t hctx, map* } int cls_cxx_map_get_keys(cls_method_context_t hctx, const string &start_obj, - uint64_t max_to_get, set *keys) + uint64_t max_to_get, set *keys, + bool *more) { PrimaryLogPG::OpContext **pctx = (PrimaryLogPG::OpContext **)hctx; vector ops(1); @@ -455,6 +458,7 @@ int cls_cxx_map_get_keys(cls_method_context_t hctx, const string &start_obj, bufferlist::iterator iter = op.outdata.begin(); try { ::decode(*keys, iter); + ::decode(*more, iter); } catch (buffer::error& err) { return -EIO; } @@ -463,7 +467,7 @@ int cls_cxx_map_get_keys(cls_method_context_t hctx, const string &start_obj, int cls_cxx_map_get_vals(cls_method_context_t hctx, const string &start_obj, const string &filter_prefix, uint64_t max_to_get, - map *vals) + map *vals, bool *more) { PrimaryLogPG::OpContext **pctx = (PrimaryLogPG::OpContext **)hctx; vector ops(1); @@ -483,6 +487,7 @@ int cls_cxx_map_get_vals(cls_method_context_t hctx, const string &start_obj, bufferlist::iterator iter = op.outdata.begin(); try { ::decode(*vals, iter); + ::decode(*more, iter); } catch (buffer::error& err) { return -EIO; } diff --git a/src/objclass/objclass.h b/src/objclass/objclass.h index 99be19beb3413..cd41153dafcf5 100644 --- a/src/objclass/objclass.h +++ b/src/objclass/objclass.h @@ -130,16 +130,19 @@ extern int cls_cxx_replace(cls_method_context_t hctx, int ofs, int len, bufferli extern int cls_cxx_snap_revert(cls_method_context_t hctx, snapid_t snapid); extern int cls_cxx_map_clear(cls_method_context_t hctx); extern int cls_cxx_map_get_all_vals(cls_method_context_t hctx, - std::map *vals); + std::map *vals, + bool *more); extern int cls_cxx_map_get_keys(cls_method_context_t hctx, const string &start_after, uint64_t max_to_get, - std::set *keys); + std::set *keys, + bool *more); extern int cls_cxx_map_get_vals(cls_method_context_t hctx, const string &start_after, const string &filter_prefix, uint64_t max_to_get, - std::map *vals); + std::map *vals, + bool *more); extern int cls_cxx_map_read_header(cls_method_context_t hctx, bufferlist *outbl); extern int cls_cxx_map_set_vals(cls_method_context_t hctx, const std::map *map); diff --git a/src/test/librados_test_stub/LibradosTestStub.cc b/src/test/librados_test_stub/LibradosTestStub.cc index fd67877bf0dbb..069f8e943b520 100644 --- a/src/test/librados_test_stub/LibradosTestStub.cc +++ b/src/test/librados_test_stub/LibradosTestStub.cc @@ -1185,27 +1185,22 @@ int cls_cxx_getxattrs(cls_method_context_t hctx, std::map *a } int cls_cxx_map_get_keys(cls_method_context_t hctx, const string &start_obj, - uint64_t max_to_get, std::set *keys) { + uint64_t max_to_get, std::set *keys, bool *more) { librados::TestClassHandler::MethodContext *ctx = reinterpret_cast(hctx); keys->clear(); std::map vals; - std::string last_key = start_obj; - do { - vals.clear(); - int r = ctx->io_ctx_impl->omap_get_vals(ctx->oid, last_key, "", 1024, - &vals); - if (r < 0) { - return r; - } + int r = ctx->io_ctx_impl->omap_get_vals2(ctx->oid, start_obj, "", max_to_get, + &vals, more); + if (r < 0) { + return r; + } - for (std::map::iterator it = vals.begin(); - it != vals.end(); ++it) { - last_key = it->first; - keys->insert(last_key); - } - } while (!vals.empty()); + for (std::map::iterator it = vals.begin(); + it != vals.end(); ++it) { + keys->insert(it->first); + } return keys->size(); } @@ -1231,11 +1226,11 @@ int cls_cxx_map_get_val(cls_method_context_t hctx, const string &key, int cls_cxx_map_get_vals(cls_method_context_t hctx, const string &start_obj, const string &filter_prefix, uint64_t max_to_get, - std::map *vals) { + std::map *vals, bool *more) { librados::TestClassHandler::MethodContext *ctx = reinterpret_cast(hctx); - int r = ctx->io_ctx_impl->omap_get_vals(ctx->oid, start_obj, filter_prefix, - max_to_get, vals); + int r = ctx->io_ctx_impl->omap_get_vals2(ctx->oid, start_obj, filter_prefix, + max_to_get, vals, more); if (r < 0) { return r; } diff --git a/src/test/librados_test_stub/TestIoCtxImpl.h b/src/test/librados_test_stub/TestIoCtxImpl.h index 2ba13fdcd3804..499a55f80f65a 100644 --- a/src/test/librados_test_stub/TestIoCtxImpl.h +++ b/src/test/librados_test_stub/TestIoCtxImpl.h @@ -107,6 +107,12 @@ class TestIoCtxImpl { const std::string &filter_prefix, uint64_t max_return, std::map *out_vals) = 0; + virtual int omap_get_vals2(const std::string& oid, + const std::string& start_after, + const std::string &filter_prefix, + uint64_t max_return, + std::map *out_vals, + bool *pmore) = 0; virtual int omap_rm_keys(const std::string& oid, const std::set& keys) = 0; virtual int omap_set(const std::string& oid, diff --git a/src/test/librados_test_stub/TestMemIoCtxImpl.cc b/src/test/librados_test_stub/TestMemIoCtxImpl.cc index 908bad32fba94..cccbe82059631 100644 --- a/src/test/librados_test_stub/TestMemIoCtxImpl.cc +++ b/src/test/librados_test_stub/TestMemIoCtxImpl.cc @@ -168,11 +168,12 @@ int TestMemIoCtxImpl::list_snaps(const std::string& oid, snap_set_t *out_snaps) } -int TestMemIoCtxImpl::omap_get_vals(const std::string& oid, +int TestMemIoCtxImpl::omap_get_vals2(const std::string& oid, const std::string& start_after, const std::string &filter_prefix, uint64_t max_return, - std::map *out_vals) { + std::map *out_vals, + bool *pmore) { if (out_vals == NULL) { return -EINVAL; } else if (m_client->is_blacklisted()) { @@ -193,6 +194,9 @@ int TestMemIoCtxImpl::omap_get_vals(const std::string& oid, RWLock::RLocker l(file->lock); TestMemCluster::FileOMaps::iterator o_it = m_pool->file_omaps.find(oid); if (o_it == m_pool->file_omaps.end()) { + if (pmore) { + *pmore = false; + } return 0; } @@ -210,9 +214,20 @@ int TestMemIoCtxImpl::omap_get_vals(const std::string& oid, } ++it; } + if (pmore) { + *pmore = (it != omap.end()); + } return 0; } +int TestMemIoCtxImpl::omap_get_vals(const std::string& oid, + const std::string& start_after, + const std::string &filter_prefix, + uint64_t max_return, + std::map *out_vals) { + return omap_get_vals2(oid, start_after, filter_prefix, max_return, out_vals, nullptr); +} + int TestMemIoCtxImpl::omap_rm_keys(const std::string& oid, const std::set& keys) { if (get_snap_read() != CEPH_NOSNAP) { diff --git a/src/test/librados_test_stub/TestMemIoCtxImpl.h b/src/test/librados_test_stub/TestMemIoCtxImpl.h index 9f63b7daee1bf..c76fb3cf8cb20 100644 --- a/src/test/librados_test_stub/TestMemIoCtxImpl.h +++ b/src/test/librados_test_stub/TestMemIoCtxImpl.h @@ -35,6 +35,12 @@ class TestMemIoCtxImpl : public TestIoCtxImpl { const std::string &filter_prefix, uint64_t max_return, std::map *out_vals) override; + int omap_get_vals2(const std::string& oid, + const std::string& start_after, + const std::string &filter_prefix, + uint64_t max_return, + std::map *out_vals, + bool *pmore) override; int omap_rm_keys(const std::string& oid, const std::set& keys) override; int omap_set(const std::string& oid, const std::map