Merge 'Allow tombstone GC in compaction to be disabled on user reques…

…t' from Raphael "Raph" Carvalho

Adding new APIs /column_family/tombstone_gc and /storage_service/tombstone_gc, that will allow for disabling tombstone garbage collection (GC) in compaction.

Mimicks existing APIs /column_family/autocompaction and /storage_service/autocompaction.

column_family variant must specify a single table only, following existing convention.

whereas the storage_service one can specify an entire keyspace, or a subset of a tables in a keyspace.

column_family API usage
-----

```
    The table name must be in keyspace:name format

    Get status:
    curl -s -X GET "http://127.0.0.1:10000/column_family/tombstone_gc/ks:cf"

    Enable GC
    curl -s -X POST "http://127.0.0.1:10000/column_family/tombstone_gc/ks:cf"

    Disable GC
    curl -s -X DELETE "http://127.0.0.1:10000/column_family/tombstone_gc/ks:cf"
```

storage_service API usage
-----

```
    Tables can be specified using a comma-separated list.

    Enable GC on keyspace
    curl -s -X POST "http://127.0.0.1:10000/storage_service/tombstone_gc/ks"

    Disable GC on keyspace
    curl -s -X DELETE "http://127.0.0.1:10000/storage_service/tombstone_gc/ks"

    Enable GC on a subset of tables
    curl -s -X POST
    "http://127.0.0.1:10000/storage_service/tombstone_gc/ks?cf=table1,table2"
```

Closes #13793

* github.com:scylladb/scylladb:
  test: Test new API for disabling tombstone GC
  test: rest_api: extract common testing code into generic functions
  Add API to disable tombstone GC in compaction
  api: storage_service: restore indentation
  api: storage_service: extract code to set attribute for a set of tables
  tests: Test new option for disabling tombstone GC in compaction
  compaction_strategy: bypass tombstone compaction if tombstone GC is disabled
  table: Allow tombstone GC in compaction to be disabled on user request

Fixes #14077

(cherry picked from commit 31e820e)
Signed-off-by: Raphael S. Carvalho <raphaelsc@scylladb.com>

Closes #14116

api/api-doc/column_family.json

@@ -437,6 +437,68 @@
             }
          ]
       },
+      {
+         "path":"/column_family/tombstone_gc/{name}",
+         "operations":[
+            {
+               "method":"GET",
+               "summary":"Check if tombstone GC is enabled for a given table",
+               "type":"boolean",
+               "nickname":"get_tombstone_gc",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"name",
+                     "description":"The table name in keyspace:name format",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"path"
+                  }
+               ]
+            },
+            {
+               "method":"POST",
+               "summary":"Enable tombstone GC for a given table",
+               "type":"void",
+               "nickname":"enable_tombstone_gc",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"name",
+                     "description":"The table name in keyspace:name format",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"path"
+                  }
+               ]
+            },
+            {
+               "method":"DELETE",
+               "summary":"Disable tombstone GC for a given table",
+               "type":"void",
+               "nickname":"disable_tombstone_gc",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"name",
+                     "description":"The table name in keyspace:name format",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"path"
+                  }
+               ]
+            }
+         ]
+      },
       {
          "path":"/column_family/estimate_keys/{name}",
          "operations":[

api/api-doc/storage_service.json

@@ -2110,6 +2110,65 @@
             }
          ]
       },
+      {
+         "path":"/storage_service/tombstone_gc/{keyspace}",
+         "operations":[
+            {
+               "method":"POST",
+               "summary":"Enable tombstone GC",
+               "type":"void",
+               "nickname":"enable_tombstone_gc",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"keyspace",
+                     "description":"The keyspace",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"path"
+                  },
+                  {
+                     "name":"cf",
+                     "description":"Comma-separated column family names",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"query"
+                  }
+               ]
+            },
+            {
+               "method":"DELETE",
+               "summary":"Disable tombstone GC",
+               "type":"void",
+               "nickname":"disable_tombstone_gc",
+               "produces":[
+                  "application/json"
+               ],
+               "parameters":[
+                  {
+                     "name":"keyspace",
+                     "description":"The keyspace",
+                     "required":true,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"path"
+                  },
+                  {
+                     "name":"cf",
+                     "description":"Comma-separated column family names",
+                     "required":false,
+                     "allowMultiple":false,
+                     "type":"string",
+                     "paramType":"query"
+                  }
+               ]
+            }
+         ]
+      },
       {
          "path":"/storage_service/deliver_hints",
          "operations":[

api/column_family.cc

@@ -892,6 +892,30 @@ void set_column_family(http_context& ctx, routes& r) {
         });
     });
 
+    cf::get_tombstone_gc.set(r, [&ctx] (const_req req) {
+        auto uuid = get_uuid(req.param["name"], ctx.db.local());
+        replica::table& t = ctx.db.local().find_column_family(uuid);
+        return t.tombstone_gc_enabled();
+    });
+
+    cf::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        apilog.info("column_family/enable_tombstone_gc: name={}", req->param["name"]);
+        return foreach_column_family(ctx, req->param["name"], [](replica::table& t) {
+            t.set_tombstone_gc_enabled(true);
+        }).then([] {
+            return make_ready_future<json::json_return_type>(json_void());
+        });
+    });
+
+    cf::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        apilog.info("column_family/disable_tombstone_gc: name={}", req->param["name"]);
+        return foreach_column_family(ctx, req->param["name"], [](replica::table& t) {
+            t.set_tombstone_gc_enabled(false);
+        }).then([] {
+            return make_ready_future<json::json_return_type>(json_void());
+        });
+    });
+
     cf::get_built_indexes.set(r, [&ctx](std::unique_ptr<request> req) {
         auto ks_cf = parse_fully_qualified_cf_name(req->param["name"]);
         auto&& ks = std::get<0>(ks_cf);

api/storage_service.cc

@@ -216,32 +216,47 @@ seastar::future<json::json_return_type> run_toppartitions_query(db::toppartition
     });
 }
 
-future<json::json_return_type> set_tables_autocompaction(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
+static future<json::json_return_type> set_tables(http_context& ctx, const sstring& keyspace, std::vector<sstring> tables, std::function<future<>(replica::table&)> set) {
     if (tables.empty()) {
         tables = map_keys(ctx.db.local().find_keyspace(keyspace).metadata().get()->cf_meta_data());
     }
 
-    apilog.info("set_tables_autocompaction: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
-    return do_with(keyspace, std::move(tables), [&ctx, enabled] (const sstring &keyspace, const std::vector<sstring>& tables) {
-        return ctx.db.invoke_on(0, [&ctx, &keyspace, &tables, enabled] (replica::database& db) {
-            auto g = replica::database::autocompaction_toggle_guard(db);
-            return ctx.db.invoke_on_all([&keyspace, &tables, enabled] (replica::database& db) {
-                return parallel_for_each(tables, [&db, &keyspace, enabled] (const sstring& table) {
-                    replica::column_family& cf = db.find_column_family(keyspace, table);
-                    if (enabled) {
-                        cf.enable_auto_compaction();
-                    } else {
-                        return cf.disable_auto_compaction();
-                    }
-                    return make_ready_future<>();
-                });
-            }).finally([g = std::move(g)] {});
+    return do_with(keyspace, std::move(tables), [&ctx, set] (const sstring& keyspace, const std::vector<sstring>& tables) {
+        return ctx.db.invoke_on_all([&keyspace, &tables, set] (replica::database& db) {
+            return parallel_for_each(tables, [&db, &keyspace, set] (const sstring& table) {
+                replica::table& t = db.find_column_family(keyspace, table);
+                return set(t);
+            });
         });
     }).then([] {
         return make_ready_future<json::json_return_type>(json_void());
     });
 }
 
+future<json::json_return_type> set_tables_autocompaction(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
+    apilog.info("set_tables_autocompaction: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
+
+    return ctx.db.invoke_on(0, [&ctx, keyspace, tables = std::move(tables), enabled] (replica::database& db) {
+        auto g = replica::database::autocompaction_toggle_guard(db);
+        return set_tables(ctx, keyspace, tables, [enabled] (replica::table& cf) {
+            if (enabled) {
+                cf.enable_auto_compaction();
+            } else {
+                return cf.disable_auto_compaction();
+            }
+            return make_ready_future<>();
+        }).finally([g = std::move(g)] {});
+    });
+}
+
+future<json::json_return_type> set_tables_tombstone_gc(http_context& ctx, const sstring &keyspace, std::vector<sstring> tables, bool enabled) {
+    apilog.info("set_tables_tombstone_gc: enabled={} keyspace={} tables={}", enabled, keyspace, tables);
+    return set_tables(ctx, keyspace, std::move(tables), [enabled] (replica::table& t) {
+        t.set_tombstone_gc_enabled(enabled);
+        return make_ready_future<>();
+    });
+}
+
 void set_transport_controller(http_context& ctx, routes& r, cql_transport::controller& ctl) {
     ss::start_native_transport.set(r, [&ctl](std::unique_ptr<request> req) {
         return smp::submit_to(0, [&] {
@@ -1130,7 +1145,23 @@ void set_storage_service(http_context& ctx, routes& r, sharded<service::storage_
         return set_tables_autocompaction(ctx, keyspace, tables, false);
     });
 
-    ss::deliver_hints.set(r, [](std::unique_ptr<request> req) {
+    ss::enable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        auto keyspace = validate_keyspace(ctx, req->param);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
+        apilog.info("enable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
+        return set_tables_tombstone_gc(ctx, keyspace, tables, true);
+    });
+
+    ss::disable_tombstone_gc.set(r, [&ctx](std::unique_ptr<http::request> req) {
+        auto keyspace = validate_keyspace(ctx, req->param);
+        auto tables = parse_tables(keyspace, ctx, req->query_parameters, "cf");
+
+        apilog.info("disable_tombstone_gc: keyspace={} tables={}", keyspace, tables);
+        return set_tables_tombstone_gc(ctx, keyspace, tables, false);
+    });
+
+    ss::deliver_hints.set(r, [](std::unique_ptr<http::request> req) {
         //TBD
         unimplemented();
         auto host = req->get_query_param("host");

compaction/compaction.cc

@@ -169,6 +169,10 @@ std::ostream& operator<<(std::ostream& os, pretty_printed_throughput tp) {
 
 static api::timestamp_type get_max_purgeable_timestamp(const table_state& table_s, sstable_set::incremental_selector& selector,
         const std::unordered_set<shared_sstable>& compacting_set, const dht::decorated_key& dk) {
+    if (!table_s.tombstone_gc_enabled()) [[unlikely]] {
+        return api::min_timestamp;
+    }
+
     auto timestamp = table_s.min_memtable_timestamp();
     std::optional<utils::hashed_key> hk;
     for (auto&& sst : boost::range::join(selector.select(dk).sstables, table_s.compacted_undeleted_sstables())) {
@@ -570,7 +574,7 @@ class compaction {
     // Tombstone expiration is enabled based on the presence of sstable set.
     // If it's not present, we cannot purge tombstones without the risk of resurrecting data.
     bool tombstone_expiration_enabled() const {
-        return bool(_sstable_set);
+        return bool(_sstable_set) && _table_s.tombstone_gc_enabled();
     }
 
     compaction_writer create_gc_compaction_writer() const {

compaction/leveled_compaction_strategy.cc

@@ -31,6 +31,10 @@ compaction_descriptor leveled_compaction_strategy::get_sstables_for_compaction(t
         return candidate;
     }
 
+    if (!table_s.tombstone_gc_enabled()) {
+        return compaction_descriptor();
+    }
+
     // if there is no sstable to compact in standard way, try compacting based on droppable tombstone ratio
     // unlike stcs, lcs can look for sstable with highest droppable tombstone ratio, so as not to choose
     // a sstable which droppable data shadow data in older sstable, by starting from highest levels which

compaction/size_tiered_compaction_strategy.cc

@@ -164,6 +164,10 @@ size_tiered_compaction_strategy::get_sstables_for_compaction(table_state& table_
         return sstables::compaction_descriptor(std::move(most_interesting), service::get_local_compaction_priority());
     }
 
+    if (!table_s.tombstone_gc_enabled()) {
+        return compaction_descriptor();
+    }
+
     // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
     // ratio is greater than threshold.
     // prefer oldest sstables from biggest size tiers because they will be easier to satisfy conditions for

compaction/table_state.hh

@@ -43,6 +43,7 @@ public:
     virtual api::timestamp_type min_memtable_timestamp() const = 0;
     virtual future<> on_compaction_completion(sstables::compaction_completion_desc desc, sstables::offstrategy offstrategy) = 0;
     virtual bool is_auto_compaction_disabled_by_user() const noexcept = 0;
+    virtual bool tombstone_gc_enabled() const noexcept = 0;
     virtual const tombstone_gc_state& get_tombstone_gc_state() const noexcept = 0;
     virtual compaction_backlog_tracker& get_backlog_tracker() = 0;
 };

compaction/time_window_compaction_strategy.cc

@@ -272,6 +272,10 @@ time_window_compaction_strategy::get_next_non_expired_sstables(table_state& tabl
         return most_interesting;
     }
 
+    if (!table_s.tombstone_gc_enabled()) {
+        return {};
+    }
+
     // if there is no sstable to compact in standard way, try compacting single sstable whose droppable tombstone
     // ratio is greater than threshold.
     auto e = boost::range::remove_if(non_expiring_sstables, [this, compaction_time, &table_s] (const shared_sstable& sst) -> bool {

replica/database.hh

@@ -436,6 +436,7 @@ private:
     sstables::sstables_manager& _sstables_manager;
     secondary_index::secondary_index_manager _index_manager;
     bool _compaction_disabled_by_user = false;
+    bool _tombstone_gc_enabled = true;
     utils::phased_barrier _flush_barrier;
     std::vector<view_ptr> _views;
 
@@ -961,6 +962,13 @@ public:
 
     void enable_auto_compaction();
     future<> disable_auto_compaction();
+
+    void set_tombstone_gc_enabled(bool tombstone_gc_enabled) noexcept;
+
+    bool tombstone_gc_enabled() const noexcept {
+        return _tombstone_gc_enabled;
+    }
+
     bool is_auto_compaction_disabled_by_user() const {
       return _compaction_disabled_by_user;
     }

replica/table.cc

@@ -2438,6 +2438,14 @@ table::disable_auto_compaction() {
     });
 }
 
+void table::set_tombstone_gc_enabled(bool tombstone_gc_enabled) noexcept {
+    _tombstone_gc_enabled = tombstone_gc_enabled;
+    tlogger.info0("Tombstone GC was {} for {}.{}", tombstone_gc_enabled ? "enabled" : "disabled", _schema->ks_name(), _schema->cf_name());
+    if (_tombstone_gc_enabled) {
+        trigger_compaction();
+    }
+}
+
 flat_mutation_reader_v2
 table::make_reader_v2_excluding_sstables(schema_ptr s,
         reader_permit permit,
@@ -2679,6 +2687,9 @@ class compaction_group::table_state : public compaction::table_state {
     bool is_auto_compaction_disabled_by_user() const noexcept override {
         return _t.is_auto_compaction_disabled_by_user();
     }
+    bool tombstone_gc_enabled() const noexcept override {
+        return _t._tombstone_gc_enabled;
+    }
     const tombstone_gc_state& get_tombstone_gc_state() const noexcept override {
         return _t.get_compaction_manager().get_tombstone_gc_state();
     }