cdc: topology_description entries as chunked_vector

Lists can grow very big. Let's use a chunked vector to prevent large contiguous allocations. Fixes: scylladb#15302.
michaelhly · Sep 15, 2023 · 4f0d60b · 4f0d60b
1 parent b87660f
commit 4f0d60b
Show file tree

Hide file tree

Showing 7 changed files with 15 additions and 15 deletions.
diff --git a/cdc/generation.cc b/cdc/generation.cc
@@ -155,18 +155,18 @@ bool token_range_description::operator==(const token_range_description& o) const
         && sharding_ignore_msb == o.sharding_ignore_msb;
 }
 
-topology_description::topology_description(std::vector<token_range_description> entries)
+topology_description::topology_description(utils::chunked_vector<token_range_description> entries)
     : _entries(std::move(entries)) {}
 
 bool topology_description::operator==(const topology_description& o) const {
     return _entries == o._entries;
 }
 
-const std::vector<token_range_description>& topology_description::entries() const& {
+const utils::chunked_vector<token_range_description>& topology_description::entries() const& {
     return _entries;
 }
 
-std::vector<token_range_description>&& topology_description::entries() && {
+utils::chunked_vector<token_range_description>&& topology_description::entries() && {
     return std::move(_entries);
 }
 
@@ -355,7 +355,7 @@ cdc::topology_description make_new_generation_description(
         const locator::token_metadata_ptr tmptr) {
     const auto tokens = get_tokens(bootstrap_tokens, tmptr);
 
-    std::vector<token_range_description> vnode_descriptions;
+    utils::chunked_vector<token_range_description> vnode_descriptions;
     vnode_descriptions.reserve(tokens.size());
 
     vnode_descriptions.push_back(create_token_range_description(0, tokens.back(), tokens.front(), get_sharding_info));

diff --git a/cdc/generation.hh b/cdc/generation.hh
@@ -92,13 +92,13 @@ struct token_range_description {
  * in the `_entries` vector. See the comment above `token_range_description` for explanation.
  */
 class topology_description {
-    std::vector<token_range_description> _entries;
+    utils::chunked_vector<token_range_description> _entries;
 public:
-    topology_description(std::vector<token_range_description> entries);
+    topology_description(utils::chunked_vector<token_range_description> entries);
     bool operator==(const topology_description&) const;
 
-    const std::vector<token_range_description>& entries() const&;
-    std::vector<token_range_description>&& entries() &&;
+    const utils::chunked_vector<token_range_description>& entries() const&;
+    utils::chunked_vector<token_range_description>&& entries() &&;
 };
 
 /**

diff --git a/cdc/metadata.cc b/cdc/metadata.cc
@@ -40,7 +40,7 @@ static cdc::stream_id get_stream(
 
 // non-static for testing
 cdc::stream_id get_stream(
-        const std::vector<cdc::token_range_description>& entries,
+        const utils::chunked_vector<cdc::token_range_description>& entries,
         dht::token tok) {
     if (entries.empty()) {
         on_internal_error(cdc_log, "get_stream: entries empty");

diff --git a/db/system_distributed_keyspace.cc b/db/system_distributed_keyspace.cc
@@ -492,7 +492,7 @@ system_distributed_keyspace::read_cdc_topology_description(
             return {};
         }
 
-        std::vector<cdc::token_range_description> entries;
+        utils::chunked_vector<cdc::token_range_description> entries;
 
         auto entries_val = value_cast<list_type_impl::native_type>(
                 cdc_generation_description_type->deserialize(cql_result->one().get_view("description")));
@@ -552,7 +552,7 @@ system_distributed_keyspace::insert_cdc_generation(
 
 future<std::optional<cdc::topology_description>>
 system_distributed_keyspace::read_cdc_generation(utils::UUID id) {
-    std::vector<cdc::token_range_description> entries;
+    utils::chunked_vector<cdc::token_range_description> entries;
     size_t num_ranges = 0;
     co_await _qp.query_internal(
             // This should be a local read so 20s should be more than enough

diff --git a/db/system_keyspace.cc b/db/system_keyspace.cc
@@ -2667,7 +2667,7 @@ future<> system_keyspace::update_topology_fence_version(int64_t value) {
 
 future<cdc::topology_description>
 system_keyspace::read_cdc_generation(utils::UUID id) {
-    std::vector<cdc::token_range_description> entries;
+    utils::chunked_vector<cdc::token_range_description> entries;
     co_await _qp.query_internal(
             format("SELECT range_end, streams, ignore_msb FROM {}.{} WHERE key = '{}' AND id = ?",
                    NAME, CDC_GENERATIONS_V3, cdc::CDC_GENERATIONS_V3_KEY),

diff --git a/docs/dev/cdc.md b/docs/dev/cdc.md
@@ -42,7 +42,7 @@ namespace cdc {
         uint8_t sharding_ignore_msb;
     };
     class topology_description {
-        std::vector<token_range_description> _entries;
+        utils::chunked_vector<token_range_description> _entries;
 public:
         ... methods ...
     };

diff --git a/test/boost/cdc_generation_test.cc b/test/boost/cdc_generation_test.cc
@@ -22,7 +22,7 @@ topology_description limit_number_of_streams_if_needed(topology_description&& de
 } // namespace cdc
 
 static cdc::topology_description create_description(const std::vector<size_t>& streams_count_per_vnode) {
-    std::vector<cdc::token_range_description> result;
+    utils::chunked_vector<cdc::token_range_description> result;
     result.reserve(streams_count_per_vnode.size());
     size_t vnode_index = 0;
     int64_t token = std::numeric_limits<int64_t>::min() + 100;
@@ -71,7 +71,7 @@ static void assert_stream_ids_in_right_token_ranges(const cdc::topology_descript
 
 }
 
-cdc::stream_id get_stream(const std::vector<cdc::token_range_description>& entries, dht::token tok);
+cdc::stream_id get_stream(const utils::chunked_vector<cdc::token_range_description>& entries, dht::token tok);
 
 static void assert_random_tokens_mapped_to_streams_with_tokens_in_the_same_token_range(const cdc::topology_description& desc) {
     for (size_t count = 0; count < 100; ++count) {