Deletes: adding configuration parameter for purging deleted cells.

test/src/unit-capi-config.cc

@@ -245,6 +245,7 @@ void check_save_to_file() {
   ss << "sm.consolidation.amplification 1.0\n";
   ss << "sm.consolidation.buffer_size 50000000\n";
   ss << "sm.consolidation.mode fragments\n";
+  ss << "sm.consolidation.purge_deleted_cells false\n";
   ss << "sm.consolidation.step_max_frags 4294967295\n";
   ss << "sm.consolidation.step_min_frags 4294967295\n";
   ss << "sm.consolidation.step_size_ratio 0.0\n";
@@ -621,6 +622,7 @@ TEST_CASE("C API: Test config iter", "[capi][config]") {
   all_param_values["sm.consolidation.timestamp_start"] = "0";
   all_param_values["sm.consolidation.timestamp_end"] =
       std::to_string(UINT64_MAX);
+  all_param_values["sm.consolidation.purge_deleted_cells"] = "false";
   all_param_values["sm.consolidation.step_min_frags"] = "4294967295";
   all_param_values["sm.consolidation.step_max_frags"] = "4294967295";
   all_param_values["sm.consolidation.buffer_size"] = "50000000";

test/src/unit-capi-consolidation.cc

@@ -4720,6 +4720,17 @@ TEST_CASE_METHOD(
   REQUIRE(rc == TILEDB_OK);
   REQUIRE(error == nullptr);
 
+  // Test purge deleted cells
+  rc = tiledb_config_set(
+      config, "sm.consolidation.purge_deleted_cells", "1", &error);
+  REQUIRE(rc == TILEDB_ERR);
+  REQUIRE(error != nullptr);
+  tiledb_error_free(&error);
+  rc = tiledb_config_set(
+      config, "sm.consolidation.purge_deleted_cells", "true", &error);
+  REQUIRE(rc == TILEDB_OK);
+  REQUIRE(error == nullptr);
+
   // Test min frags
   rc = tiledb_config_set(
       config, "sm.consolidation.step_min_frags", "-1", &error);

tiledb/sm/c_api/tiledb.h

@@ -1101,6 +1101,10 @@ TILEDB_EXPORT void tiledb_config_free(tiledb_config_t** config) TILEDB_NOEXCEPT;
  *    The number of consolidation steps to be performed when executing
  *    the consolidation algorithm.<br>
  *    **Default**: 1
+ * - `sm.consolidation.purge_deleted_cells` <br>
+ *    **Experimental** <br>
+ *    Purge deleted cells from the consolidated fragment or not.<br>
+ *    **Default**: false
  * - `sm.consolidation.step_min_frags` <br>
  *    The minimum number of fragments to consolidate in a single step.<br>
  *    **Default**: UINT32_MAX

tiledb/sm/config/config.cc

@@ -106,6 +106,7 @@ const std::string Config::SM_IO_CONCURRENCY_LEVEL =
 const std::string Config::SM_SKIP_CHECKSUM_VALIDATION = "false";
 const std::string Config::SM_CONSOLIDATION_AMPLIFICATION = "1.0";
 const std::string Config::SM_CONSOLIDATION_BUFFER_SIZE = "50000000";
+const std::string Config::SM_CONSOLIDATION_PURGE_DELETED_CELLS = "false";
 const std::string Config::SM_CONSOLIDATION_STEPS = "4294967295";
 const std::string Config::SM_CONSOLIDATION_STEP_MIN_FRAGS = "4294967295";
 const std::string Config::SM_CONSOLIDATION_STEP_MAX_FRAGS = "4294967295";
@@ -277,6 +278,8 @@ Config::Config() {
   param_values_["sm.consolidation.amplification"] =
       SM_CONSOLIDATION_AMPLIFICATION;
   param_values_["sm.consolidation.buffer_size"] = SM_CONSOLIDATION_BUFFER_SIZE;
+  param_values_["sm.consolidation.purge_deleted_cells"] =
+      SM_CONSOLIDATION_PURGE_DELETED_CELLS;
   param_values_["sm.consolidation.step_min_frags"] =
       SM_CONSOLIDATION_STEP_MIN_FRAGS;
   param_values_["sm.consolidation.step_max_frags"] =
@@ -616,6 +619,9 @@ Status Config::unset(const std::string& param) {
   } else if (param == "sm.consolidation.buffer_size") {
     param_values_["sm.consolidation.buffer_size"] =
         SM_CONSOLIDATION_BUFFER_SIZE;
+  } else if (param == "sm.consolidation.purge_deleted_cells") {
+    param_values_["sm.consolidation.steps"] =
+        SM_CONSOLIDATION_PURGE_DELETED_CELLS;
   } else if (param == "sm.consolidation.steps") {
     param_values_["sm.consolidation.steps"] = SM_CONSOLIDATION_STEPS;
   } else if (param == "sm.consolidation.step_min_frags") {
@@ -854,6 +860,8 @@ Status Config::sanity_check(
     RETURN_NOT_OK(utils::parse::convert(value, &vf));
   } else if (param == "sm.consolidation.buffer_size") {
     RETURN_NOT_OK(utils::parse::convert(value, &vuint64));
+  } else if (param == "sm.consolidation.purge_deleted_cells") {
+    RETURN_NOT_OK(utils::parse::convert(value, &v));
   } else if (param == "sm.consolidation.steps") {
     RETURN_NOT_OK(utils::parse::convert(value, &v32));
   } else if (param == "sm.consolidation.step_min_frags") {

tiledb/sm/config/config.h

@@ -235,6 +235,9 @@ class Config {
   /** The buffer size for each attribute used in consolidation. */
   static const std::string SM_CONSOLIDATION_BUFFER_SIZE;
 
+  /** Purge deleted cells or not. */
+  static const std::string SM_CONSOLIDATION_PURGE_DELETED_CELLS;
+
   /** Number of steps in the consolidation algorithm. */
   static const std::string SM_CONSOLIDATION_STEPS;
 

tiledb/sm/consolidator/fragment_consolidator.cc

@@ -820,6 +820,12 @@ Status FragmentConsolidator::set_config(const Config* config) {
   RETURN_NOT_OK(merged_config.get<float>(
       "sm.consolidation.step_size_ratio", &config_.size_ratio_, &found));
   assert(found);
+  config_.purge_deleted_cells_ = false;
+  RETURN_NOT_OK(merged_config.get<bool>(
+      "sm.consolidation.purge_deleted_cells",
+      &config_.purge_deleted_cells_,
+      &found));
+  assert(found);
   config_.min_frags_ = 0;
   RETURN_NOT_OK(merged_config.get<uint32_t>(
       "sm.consolidation.step_min_frags", &config_.min_frags_, &found));

tiledb/sm/consolidator/fragment_consolidator.h

@@ -166,6 +166,8 @@ class FragmentConsolidator : public Consolidator {
     float size_ratio_;
     /** Is the refactored reader in use or not */
     bool use_refactored_reader_;
+    /** Purge deleted cells or not. */
+    bool purge_deleted_cells_;
   };
 
   /* ********************************* */

tiledb/sm/cpp_api/config.h

@@ -333,6 +333,10 @@ class Config {
    *    The number of consolidation steps to be performed when executing
    *    the consolidation algorithm.<br>
    *    **Default**: 1
+   * - `sm.consolidation.purge_deleted_cells` <br>
+   *    **Experimental** <br>
+   *    Purge deleted cells from the consolidated fragment or not.<br>
+   *    **Default**: false
    * - `sm.consolidation.step_min_frags` <br>
    *    The minimum number of fragments to consolidate in a single step.<br>
    *    **Default**: UINT32_MAX