Deletes: implement delete strategy.

This implements the strategy for delete queries that allow to write the
delete commit to disk.

It also implements a storage manager API to load the deletes from disk,
which will be used by the readers.

---
TYPE: IMPROVEMENT
DESC: Deletes: implement delete strategy.

test/CMakeLists.txt

@@ -203,6 +203,7 @@ if (TILEDB_CPP_API)
     src/unit-cppapi-consolidation.cc
     src/unit-cppapi-consolidation-with-timestamps.cc
     src/unit-cppapi-datetimes.cc
+    src/unit-cppapi-deletes.cc
     src/unit-cppapi-time.cc
     src/unit-cppapi-fill_values.cc
     src/unit-cppapi-filter.cc

test/src/unit-cppapi-consolidation-with-timestamps.cc

@@ -27,7 +27,7 @@
  *
  * @section DESCRIPTION
  *
- * Tests the CPP API consolidation with timestamps.
+ * Tests the CPP API for consolidation with timestamps.
  */
 
 #include "catch.hpp"

test/src/unit-cppapi-deletes.cc

@@ -0,0 +1,294 @@
+/**
+ * @file   unit-cppapi-deletes.cc
+ *
+ * @section LICENSE
+ *
+ * The MIT License
+ *
+ * @copyright Copyright (c) 2022 TileDB Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ * @section DESCRIPTION
+ *
+ * Tests the CPP API for deletes.
+ */
+
+#include "catch.hpp"
+#include "test/src/ast_helpers.h"
+#include "test/src/helpers.h"
+#include "tiledb/sm/array/array_directory.h"
+#include "tiledb/sm/c_api/tiledb_struct_def.h"
+#include "tiledb/sm/cpp_api/tiledb"
+
+using namespace tiledb;
+using namespace tiledb::test;
+
+/** Tests for C API deletes. */
+struct DeletesFx {
+  // Constants.
+  const char* SPARSE_ARRAY_NAME = "test_deletes_array";
+
+  // TileDB context.
+  Context ctx_;
+  VFS vfs_;
+  sm::StorageManager* sm_;
+
+  // Constructors/destructors.
+  DeletesFx();
+  ~DeletesFx();
+
+  // Functions.
+  void set_legacy();
+  void create_sparse_array(bool allows_dups = false);
+  void write_sparse(
+      std::vector<int> a1,
+      std::vector<uint64_t> dim1,
+      std::vector<uint64_t> dim2,
+      uint64_t timestamp);
+  void read_sparse(
+      std::vector<int>& a1,
+      std::vector<uint64_t>& dim1,
+      std::vector<uint64_t>& dim2,
+      std::string& stats,
+      tiledb_layout_t layout,
+      uint64_t timestamp,
+      std::vector<uint64_t>* timestamps_ptr = nullptr);
+  void write_delete_condition(
+      QueryCondition& qc, uint64_t timestamp, bool error_expected = false);
+  void check_delete_conditions(
+      std::vector<QueryCondition> qcs, uint64_t timestamp);
+  void remove_sparse_array();
+  void remove_array(const std::string& array_name);
+  bool is_array(const std::string& array_name);
+};
+
+DeletesFx::DeletesFx()
+    : vfs_(ctx_) {
+  Config config;
+  config.set("sm.consolidation.buffer_size", "1000");
+  ctx_ = Context(config);
+  sm_ = ctx_.ptr().get()->ctx_->storage_manager();
+  vfs_ = VFS(ctx_);
+}
+
+DeletesFx::~DeletesFx() {
+}
+
+void DeletesFx::set_legacy() {
+  Config config;
+  config.set("sm.consolidation.buffer_size", "1000");
+  config.set("sm.query.sparse_global_order.reader", "legacy");
+  config.set("sm.query.sparse_unordered_with_dups.reader", "legacy");
+
+  ctx_ = Context(config);
+  sm_ = ctx_.ptr().get()->ctx_->storage_manager();
+  vfs_ = VFS(ctx_);
+}
+
+void DeletesFx::create_sparse_array(bool allows_dups) {
+  // Create dimensions.
+  auto d1 = Dimension::create<uint64_t>(ctx_, "d1", {{1, 4}}, 2);
+  auto d2 = Dimension::create<uint64_t>(ctx_, "d2", {{1, 4}}, 2);
+
+  // Create domain.
+  Domain domain(ctx_);
+  domain.add_dimension(d1);
+  domain.add_dimension(d2);
+
+  // Create attributes.
+  auto a1 = Attribute::create<int32_t>(ctx_, "a1");
+
+  // Create array schmea.
+  ArraySchema schema(ctx_, TILEDB_SPARSE);
+  schema.set_domain(domain);
+  schema.set_capacity(20);
+  schema.add_attributes(a1);
+
+  if (allows_dups) {
+    schema.set_allows_dups(true);
+  }
+
+  // Set up filters.
+  Filter filter(ctx_, TILEDB_FILTER_NONE);
+  FilterList filter_list(ctx_);
+  filter_list.add_filter(filter);
+  schema.set_coords_filter_list(filter_list);
+
+  Array::create(SPARSE_ARRAY_NAME, schema);
+}
+
+void DeletesFx::write_sparse(
+    std::vector<int> a1,
+    std::vector<uint64_t> dim1,
+    std::vector<uint64_t> dim2,
+    uint64_t timestamp) {
+  // Open array.
+  Array array(ctx_, SPARSE_ARRAY_NAME, TILEDB_WRITE, timestamp);
+
+  // Create query.
+  Query query(ctx_, array, TILEDB_WRITE);
+  query.set_layout(TILEDB_GLOBAL_ORDER);
+  query.set_data_buffer("a1", a1);
+  query.set_data_buffer("d1", dim1);
+  query.set_data_buffer("d2", dim2);
+
+  // Submit/finalize the query.
+  query.submit();
+  query.finalize();
+
+  // Close array.
+  array.close();
+}
+
+void DeletesFx::read_sparse(
+    std::vector<int>& a1,
+    std::vector<uint64_t>& dim1,
+    std::vector<uint64_t>& dim2,
+    std::string& stats,
+    tiledb_layout_t layout,
+    uint64_t timestamp,
+    std::vector<uint64_t>* timestamps_ptr) {
+  // Open array.
+  Array array(ctx_, SPARSE_ARRAY_NAME, TILEDB_READ, timestamp);
+
+  // Create query.
+  Query query(ctx_, array, TILEDB_READ);
+  query.set_layout(layout);
+  query.set_data_buffer("a1", a1);
+  query.set_data_buffer("d1", dim1);
+  query.set_data_buffer("d2", dim2);
+  if (timestamps_ptr != nullptr) {
+    query.set_data_buffer(tiledb_timestamps(), *timestamps_ptr);
+  }
+
+  // Submit the query.
+  query.submit();
+  CHECK(query.query_status() == Query::Status::COMPLETE);
+
+  // Get the query stats.
+  stats = query.stats();
+
+  // Close array.
+  array.close();
+}
+
+void DeletesFx::write_delete_condition(
+    QueryCondition& qc, uint64_t timestamp, bool error_expected) {
+  // Open array.
+  Array array(ctx_, SPARSE_ARRAY_NAME, TILEDB_DELETE, timestamp);
+
+  // Create query.
+  Query query(ctx_, array, TILEDB_DELETE);
+
+  query.set_condition(qc);
+
+  try {
+    query.submit();
+  } catch (std::exception) {
+    CHECK(error_expected);
+  }
+
+  CHECK(
+      query.query_status() ==
+      (error_expected ? Query::Status::FAILED : Query::Status::COMPLETE));
+
+  // Close array.
+  array.close();
+}
+
+void DeletesFx::check_delete_conditions(
+    std::vector<QueryCondition> qcs, uint64_t timestamp) {
+  // Open the array for read.
+  Array array(ctx_, SPARSE_ARRAY_NAME, TILEDB_READ, timestamp);
+  auto array_ptr = array.ptr()->array_;
+  auto& array_dir = array_ptr->array_directory();
+  auto& enc_key = *array_ptr->encryption_key();
+
+  // Load delete conditions.
+  auto&& [st, delete_conditions] =
+      sm_->load_delete_conditions(array_dir, enc_key);
+  REQUIRE(st.ok());
+  REQUIRE(delete_conditions->size() == qcs.size());
+
+  for (uint64_t i = 0; i < qcs.size(); i++) {
+    // Compare to negated condition.
+    auto cmp = qcs[i].ptr()->query_condition_->negated_condition();
+    CHECK(tiledb::test::ast_equal(delete_conditions->at(i).ast(), cmp.ast()));
+  }
+}
+
+void DeletesFx::remove_array(const std::string& array_name) {
+  if (!is_array(array_name))
+    return;
+
+  vfs_.remove_dir(array_name);
+}
+
+void DeletesFx::remove_sparse_array() {
+  remove_array(SPARSE_ARRAY_NAME);
+}
+
+bool DeletesFx::is_array(const std::string& array_name) {
+  return vfs_.is_dir(array_name);
+}
+
+TEST_CASE_METHOD(
+    DeletesFx,
+    "CPP API: Test writting delete condition",
+    "[cppapi][deletes][write-check]") {
+  remove_sparse_array();
+  create_sparse_array();
+
+  // Define query condition (a1 < 4.0).
+  QueryCondition qc(ctx_);
+  int32_t val = 4;
+  qc.init("a1", &val, sizeof(int32_t), TILEDB_LT);
+
+  // Define query condition (a1 > 8.0).
+  QueryCondition qc2(ctx_);
+  int32_t val2 = 8;
+  qc2.init("a1", &val2, sizeof(int32_t), TILEDB_GT);
+
+  write_delete_condition(qc, 1);
+  check_delete_conditions({qc}, 2);
+
+  write_delete_condition(qc2, 3);
+  check_delete_conditions({qc}, 2);
+  check_delete_conditions({qc, qc2}, 4);
+
+  remove_sparse_array();
+}
+
+TEST_CASE_METHOD(
+    DeletesFx,
+    "CPP API: Test writting invalid delete condition",
+    "[cppapi][deletes][write-check][invalid]") {
+  remove_sparse_array();
+  create_sparse_array();
+
+  // Define query condition (b < 4.0).
+  QueryCondition qc(ctx_);
+  int32_t val = 4;
+  qc.init("b", &val, sizeof(int32_t), TILEDB_LT);
+
+  write_delete_condition(qc, 1, true);
+
+  remove_sparse_array();
+}

tiledb/CMakeLists.txt

@@ -217,6 +217,8 @@ set(TILEDB_CORE_SOURCES
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/win_constants.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/misc/work_arounds.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/ast/query_ast.cc
+  ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes/deletes.cc
+  ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/deletes/serialization.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/hilbert_order.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/legacy/cell_slab_iter.cc
   ${TILEDB_CORE_INCLUDE_DIR}/tiledb/sm/query/legacy/reader.cc

tiledb/common/exception/status.h

@@ -478,6 +478,10 @@ inline Status Status_TaskError(const std::string& msg) {
 inline Status Status_RangeError(const std::string& msg) {
   return {"[TileDB::Range] Error", msg};
 }
+/** Return a Writer error class Status with a given message **/
+inline Status Status_DeletesError(const std::string& msg) {
+  return {"[TileDB::Deletes] Error", msg};
+}
 }  // namespace common
 }  // namespace tiledb
 

tiledb/sm/array/array.cc

@@ -363,6 +363,8 @@ Status Array::close() {
       RETURN_NOT_OK(storage_manager_->array_close_for_reads(this));
     } else if (query_type_ == QueryType::WRITE) {
       RETURN_NOT_OK(storage_manager_->array_close_for_writes(this));
+    } else if (query_type_ == QueryType::DELETE) {
+      RETURN_NOT_OK(storage_manager_->array_close_for_deletes(this));
     } else {
       return LOG_STATUS(
           Status_ArrayError("Error closing array; Unsupported query type."));

tiledb/sm/array/array_directory.cc

@@ -454,9 +454,17 @@ ArrayDirectory::load_commits_dir_uris_v12_or_higher(
     } else if (stdx::string::ends_with(
                    commits_dir_uris[i].to_string(),
                    constants::delete_file_suffix)) {
-      if (consolidated_commit_uris_set_.count(commits_dir_uris[i].c_str()) ==
-          0) {
-        delete_tiles_location_.emplace_back(commits_dir_uris[i], 0);
+      // Get the start and end timestamp for this fragment
+      std::pair<uint64_t, uint64_t> fragment_timestamp_range;
+      RETURN_NOT_OK_TUPLE(
+          utils::parse::get_timestamp_range(
+              commits_dir_uris[i], &fragment_timestamp_range),
+          nullopt);
+      if (timestamps_overlap(fragment_timestamp_range, false)) {
+        if (consolidated_commit_uris_set_.count(commits_dir_uris[i].c_str()) ==
+            0) {
+          delete_tiles_location_.emplace_back(commits_dir_uris[i], 0);
+        }
       }
     }
   }

tiledb/sm/array/array_directory.h

@@ -173,11 +173,11 @@ class ArrayDirectory {
     /* ********************************* */
     /*                API                */
     /* ********************************* */
-    inline URI& Uri() {
+    inline const URI& uri() const {
       return uri_;
     }
 
-    inline uint64_t Offset() {
+    inline uint64_t offset() const {
       return offset_;
     }
 

tiledb/sm/array_schema/array_schema.cc

@@ -457,7 +457,7 @@ bool ArraySchema::is_dim(const std::string& name) const {
 
 bool ArraySchema::is_field(const std::string& name) const {
   return is_attr(name) || is_dim(name) || name == constants::coords ||
-         name == constants::timestamps;
+         name == constants::timestamps || name == constants::timestamps;
 }
 
 bool ArraySchema::is_nullable(const std::string& name) const {

tiledb/sm/buffer/buffer.cc

@@ -214,10 +214,6 @@ void* Buffer::data(const uint64_t offset) const {
   return data + offset;
 }
 
-void Buffer::disown_data() {
-  owns_data_ = false;
-}
-
 uint64_t Buffer::free_space() const {
   assert(alloced_size_ >= size_);
   return alloced_size_ - size_;