From e2583f9e97a0017b736bfaef7476e747525cdc16 Mon Sep 17 00:00:00 2001 From: Luc Rancourt Date: Sat, 13 Aug 2022 14:26:40 +0300 Subject: [PATCH 1/3] Deletes: implement serialization. This implements serialization for delete queries. --- TYPE: IMPROVEMENT DESC: Deletes: implement serialization. --- test/src/unit-capi-serialized_queries.cc | 117 ++++++- .../sm/query/deletes_and_updates/deletes.cc | 5 +- tiledb/sm/query/deletes_and_updates/deletes.h | 3 +- tiledb/sm/query/query.cc | 3 +- .../serialization/posix/tiledb-rest.capnp.c++ | 206 ++++++++---- .../serialization/posix/tiledb-rest.capnp.h | 304 +++++++++++++++++- tiledb/sm/serialization/query.cc | 73 ++++- tiledb/sm/serialization/tiledb-rest.capnp | 13 + .../serialization/win32/tiledb-rest.capnp.c++ | 206 ++++++++---- .../serialization/win32/tiledb-rest.capnp.h | 304 +++++++++++++++++- 10 files changed, 1094 insertions(+), 140 deletions(-) diff --git a/test/src/unit-capi-serialized_queries.cc b/test/src/unit-capi-serialized_queries.cc index f32d713eaac..ceb8e4e3af4 100644 --- a/test/src/unit-capi-serialized_queries.cc +++ b/test/src/unit-capi-serialized_queries.cc @@ -141,6 +141,17 @@ struct SerializationFx { REQUIRE(loop_num->second > 0); } + static void check_delete_stats(const Query& query) { + auto stats = ((sm::Reader*)query.ptr()->query_->strategy())->stats(); + REQUIRE(stats != nullptr); + auto counters = stats->counters(); + REQUIRE(counters != nullptr); + auto dowork_num = counters->find( + "Context.StorageManager.Query.Deletes.dowork.timer_count"); + REQUIRE((dowork_num != counters->end())); + REQUIRE(dowork_num->second > 0); + } + void create_array(tiledb_array_type_t type) { ArraySchema schema(ctx, type); Domain domain(ctx); @@ -317,6 +328,33 @@ struct SerializationFx { check_write_stats(query); } + void write_sparse_delete() { + Array array(ctx, array_uri, TILEDB_DELETE); + Query query(ctx, array); + + // Define query condition (a1 < 5). + QueryCondition qc(ctx); + int32_t val = 5; + qc.init("a1", &val, sizeof(int32_t), TILEDB_LT); + query.set_condition(qc); + + // Serialize into a copy and submit. + std::vector serialized; + serialize_query(ctx, query, &serialized, true); + Array array2(ctx, array_uri, TILEDB_DELETE); + Query query2(ctx, array2); + deserialize_query(ctx, serialized, &query2, false); + query2.submit(); + + // Make sure query2 has logged stats + check_delete_stats(query2); + serialize_query(ctx, query2, &serialized, false); + deserialize_query(ctx, serialized, &query, true); + + // The deserialized query should also include the delete stats + check_delete_stats(query); + } + void write_sparse_array_split_coords() { std::vector d1 = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}; @@ -586,7 +624,7 @@ TEST_CASE_METHOD( deserialize_query(ctx, serialized, &query, true); REQUIRE(query.query_status() == Query::Status::COMPLETE); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); auto result_el = query.result_buffer_elements_nullable(); @@ -649,7 +687,7 @@ TEST_CASE_METHOD( deserialize_query(ctx, serialized, &query, true); REQUIRE(query.query_status() == Query::Status::COMPLETE); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); // We expect all cells where `a1` >= `cmp_value` to be filtered @@ -728,7 +766,7 @@ TEST_CASE_METHOD( deserialize_query(ctx, serialized, &query, true); REQUIRE(query.query_status() == Query::Status::COMPLETE); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); auto result_el = query.result_buffer_elements_nullable(); @@ -796,7 +834,7 @@ TEST_CASE_METHOD( // Deserialize into original query (client side). deserialize_query(ctx, serialized, &q, true); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); for (void* b : to_free) @@ -888,7 +926,7 @@ TEST_CASE_METHOD( deserialize_query(ctx, serialized, &query, true); REQUIRE(query.query_status() == Query::Status::COMPLETE); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); auto result_el = query.result_buffer_elements_nullable(); @@ -948,7 +986,7 @@ TEST_CASE_METHOD( deserialize_query(ctx, serialized, &query, true); REQUIRE(query.query_status() == Query::Status::COMPLETE); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); auto result_el = query.result_buffer_elements_nullable(); @@ -1012,7 +1050,7 @@ TEST_CASE_METHOD( deserialize_query(ctx, serialized, &query, true); REQUIRE(query.query_status() == Query::Status::COMPLETE); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); auto result_el = query.result_buffer_elements_nullable(); @@ -1067,7 +1105,7 @@ TEST_CASE_METHOD( deserialize_query(ctx, serialized, &query, true); REQUIRE(query.query_status() == Query::Status::COMPLETE); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); auto result_el = query.result_buffer_elements_nullable(); @@ -1124,7 +1162,7 @@ TEST_CASE_METHOD( // Deserialize into original query (client side). deserialize_query(ctx, serialized, &q, true); - // The deserialized query should also include the write stats + // The deserialized query should also include the read stats check_read_stats(query); for (void* b : to_free) @@ -1170,3 +1208,64 @@ TEST_CASE_METHOD( // TODO: check results } } + +TEST_CASE_METHOD( + SerializationFx, + "Query serialization, sparse delete", + "[query][sparse][delete][serialization]") { + create_array(TILEDB_SPARSE); + write_sparse_array(); + write_sparse_delete(); + + SECTION("- Read all") { + Array array(ctx, array_uri, TILEDB_READ); + Query query(ctx, array); + std::vector coords(1000); + std::vector a1(1000); + std::vector a2(1000); + std::vector a2_nullable(1000); + std::vector a3_data(1000 * 100); + std::vector a3_offsets(1000); + std::vector subarray = {1, 10, 1, 10}; + + query.set_subarray(subarray); + query.set_coordinates(coords); + query.set_data_buffer("a1", a1); + query.set_data_buffer("a2", a2); + query.set_validity_buffer("a2", a2_nullable); + query.set_data_buffer("a3", a3_data); + query.set_offsets_buffer("a3", a3_offsets); + + // Serialize into a copy and submit. + std::vector serialized; + serialize_query(ctx, query, &serialized, true); + Array array2(ctx, array_uri, TILEDB_READ); + Query query2(ctx, array2); + deserialize_query(ctx, serialized, &query2, false); + auto to_free = allocate_query_buffers(ctx, array2, &query2); + query2.submit(); + serialize_query(ctx, query2, &serialized, false); + + // Make sure query2 has logged stats + check_read_stats(query2); + + // Deserialize into original query + deserialize_query(ctx, serialized, &query, true); + REQUIRE(query.query_status() == Query::Status::COMPLETE); + + // The deserialized query should also include the read stats + check_read_stats(query); + + auto result_el = query.result_buffer_elements_nullable(); + REQUIRE(std::get<1>(result_el["a1"]) == 5); + REQUIRE(std::get<1>(result_el["a2"]) == 10); + REQUIRE(std::get<2>(result_el["a2"]) == 5); + REQUIRE(std::get<0>(result_el["a3"]) == 5); + REQUIRE(std::get<1>(result_el["a3"]) == 40); + + // TODO: check results + + for (void* b : to_free) + std::free(b); + } +} \ No newline at end of file diff --git a/tiledb/sm/query/deletes_and_updates/deletes.cc b/tiledb/sm/query/deletes_and_updates/deletes.cc index c95d0407ea4..fcc31461f3d 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes.cc +++ b/tiledb/sm/query/deletes_and_updates/deletes.cc @@ -63,7 +63,8 @@ Deletes::Deletes( std::unordered_map& buffers, Subarray& subarray, Layout layout, - QueryCondition& condition) + QueryCondition& condition, + bool skip_checks_serialization) : StrategyBase( stats, logger->clone("Deletes", ++logger_id_), @@ -94,7 +95,7 @@ Deletes::Deletes( "Cannot initialize deletes; Subarrays are not supported"); } - if (condition_.empty()) { + if (!skip_checks_serialization && condition_.empty()) { throw DeleteStatusException( "Cannot initialize deletes; One condition is needed"); } diff --git a/tiledb/sm/query/deletes_and_updates/deletes.h b/tiledb/sm/query/deletes_and_updates/deletes.h index c7b7b3e1e95..d4058dd7313 100644 --- a/tiledb/sm/query/deletes_and_updates/deletes.h +++ b/tiledb/sm/query/deletes_and_updates/deletes.h @@ -60,7 +60,8 @@ class Deletes : public StrategyBase, public IQueryStrategy { std::unordered_map& buffers, Subarray& subarray, Layout layout, - QueryCondition& condition); + QueryCondition& condition, + bool skip_checks_serialization = false); /** Destructor. */ ~Deletes(); diff --git a/tiledb/sm/query/query.cc b/tiledb/sm/query/query.cc index 58b8dc3d5d1..ce6f53c2c8d 100644 --- a/tiledb/sm/query/query.cc +++ b/tiledb/sm/query/query.cc @@ -1350,7 +1350,8 @@ Status Query::create_strategy(bool skip_checks_serialization) { buffers_, subarray_, layout_, - condition_)); + condition_, + skip_checks_serialization)); } else { return logger_->status( Status_QueryError("Cannot create strategy; unsupported query type")); diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ index 3b1451962e7..bdb4589c90b 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.c++ @@ -3966,6 +3966,70 @@ const ::capnp::_::RawSchema s_e19754f813ccf79c = { 4, 5, i_e19754f813ccf79c, nullptr, nullptr, { &s_e19754f813ccf79c, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_d74f5fed155d316c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 108, 49, 93, 21, 237, 95, 79, 215, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 101, 108, 101, 116, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 100, 105, 116, 105, 111, + 110, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 111, 192, 31, 135, 185, 124, 245, 234, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_d74f5fed155d316c = b_d74f5fed155d316c.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_d74f5fed155d316c[] = { + &s_c7e036a11506a1a4, + &s_eaf57cb9871fc06f, +}; +static const uint16_t m_d74f5fed155d316c[] = {0, 1}; +static const uint16_t i_d74f5fed155d316c[] = {0, 1}; +const ::capnp::_::RawSchema s_d74f5fed155d316c = { + 0xd74f5fed155d316c, b_d74f5fed155d316c.words, 49, d_d74f5fed155d316c, m_d74f5fed155d316c, + 2, 2, i_d74f5fed155d316c, nullptr, nullptr, { &s_d74f5fed155d316c, nullptr, nullptr, 0, 0, nullptr } +}; +#endif // !CAPNP_LITE static const ::capnp::_::AlignedData<79> b_def87cead82188e7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 231, 136, 33, 216, 234, 124, 248, 222, @@ -4319,143 +4383,150 @@ const ::capnp::_::RawSchema s_9b9a5fc7713a8692 = { 4, 5, i_9b9a5fc7713a8692, nullptr, nullptr, { &s_9b9a5fc7713a8692, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<294> b_96ba49d0f8b23ccc = { +static const ::capnp::_::AlignedData<309> b_96ba49d0f8b23ccc = { { 0, 0, 0, 0, 5, 0, 6, 0, 204, 60, 178, 248, 208, 73, 186, 150, 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 12, 0, 7, 0, 0, 0, 0, 0, + 13, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 191, 3, 0, 0, + 25, 0, 0, 0, 247, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 68, 0, 0, 0, 3, 0, 4, 0, + 72, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 1, 0, 0, 186, 0, 0, 0, + 233, 1, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 1, 0, 0, 3, 0, 1, 0, - 236, 1, 0, 0, 2, 0, 1, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 8, 2, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 1, 0, 0, 58, 0, 0, 0, + 5, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 228, 1, 0, 0, 3, 0, 1, 0, - 240, 1, 0, 0, 2, 0, 1, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 237, 1, 0, 0, 58, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 1, 0, 0, 3, 0, 1, 0, - 244, 1, 0, 0, 2, 0, 1, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 1, 0, 0, 42, 0, 0, 0, + 13, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 236, 1, 0, 0, 3, 0, 1, 0, - 248, 1, 0, 0, 2, 0, 1, 0, + 8, 2, 0, 0, 3, 0, 1, 0, + 20, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 1, 0, 0, 58, 0, 0, 0, + 17, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 1, 0, 0, 3, 0, 1, 0, - 252, 1, 0, 0, 2, 0, 1, 0, + 12, 2, 0, 0, 3, 0, 1, 0, + 24, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 1, 0, 0, 58, 0, 0, 0, + 21, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 244, 1, 0, 0, 3, 0, 1, 0, - 0, 2, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 1, 0, 0, 50, 0, 0, 0, + 25, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 248, 1, 0, 0, 3, 0, 1, 0, - 4, 2, 0, 0, 2, 0, 1, 0, + 20, 2, 0, 0, 3, 0, 1, 0, + 32, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 2, 0, 0, 226, 0, 0, 0, + 29, 2, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 2, 0, 0, 3, 0, 1, 0, - 20, 2, 0, 0, 2, 0, 1, 0, + 36, 2, 0, 0, 3, 0, 1, 0, + 48, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 2, 0, 0, 186, 0, 0, 0, + 45, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 2, 0, 0, 3, 0, 1, 0, - 32, 2, 0, 0, 2, 0, 1, 0, + 48, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 2, 0, 0, 202, 0, 0, 0, + 57, 2, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 36, 2, 0, 0, 3, 0, 1, 0, - 48, 2, 0, 0, 2, 0, 1, 0, + 64, 2, 0, 0, 3, 0, 1, 0, + 76, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 2, 0, 0, 122, 0, 0, 0, + 73, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 44, 2, 0, 0, 3, 0, 1, 0, - 56, 2, 0, 0, 2, 0, 1, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 192, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 2, 0, 0, 210, 0, 0, 0, + 81, 2, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 60, 2, 0, 0, 3, 0, 1, 0, - 72, 2, 0, 0, 2, 0, 1, 0, + 88, 2, 0, 0, 3, 0, 1, 0, + 100, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 2, 0, 0, 146, 0, 0, 0, + 97, 2, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 2, 0, 0, 3, 0, 1, 0, - 84, 2, 0, 0, 2, 0, 1, 0, + 100, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 2, 0, 0, 58, 0, 0, 0, + 109, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 2, 0, 0, 3, 0, 1, 0, - 88, 2, 0, 0, 2, 0, 1, 0, + 104, 2, 0, 0, 3, 0, 1, 0, + 116, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 85, 2, 0, 0, 50, 0, 0, 0, + 113, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 80, 2, 0, 0, 3, 0, 1, 0, - 92, 2, 0, 0, 2, 0, 1, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 120, 2, 0, 0, 2, 0, 1, 0, 15, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 89, 2, 0, 0, 98, 0, 0, 0, + 117, 2, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 88, 2, 0, 0, 3, 0, 1, 0, - 100, 2, 0, 0, 2, 0, 1, 0, + 116, 2, 0, 0, 3, 0, 1, 0, + 128, 2, 0, 0, 2, 0, 1, 0, 16, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 2, 0, 0, 98, 0, 0, 0, + 125, 2, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 2, 0, 0, 3, 0, 1, 0, - 108, 2, 0, 0, 2, 0, 1, 0, + 124, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 2, 0, 0, 3, 0, 1, 0, + 140, 2, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 66, 117, 102, 102, 101, 114, 72, 101, 97, 100, 101, 114, 115, 0, 0, @@ -4611,6 +4682,14 @@ static const ::capnp::_::AlignedData<294> b_96ba49d0f8b23ccc = { 156, 247, 204, 19, 248, 84, 151, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 108, 101, 116, 101, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 108, 49, 93, 21, 237, 95, 79, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -4624,13 +4703,14 @@ static const ::capnp::_::RawSchema* const d_96ba49d0f8b23ccc[] = { &s_b6c95b4b8111ad36, &s_c7e036a11506a1a4, &s_d20a578112fa92a2, + &s_d74f5fed155d316c, &s_e19754f813ccf79c, }; -static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 13, 16, 1, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4}; -static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; +static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 13, 17, 16, 1, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4}; +static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; const ::capnp::_::RawSchema s_96ba49d0f8b23ccc = { - 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 294, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, - 7, 17, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr } + 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 309, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, + 8, 18, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<69> b_9df6f2a42c4e5f0b = { @@ -6254,6 +6334,14 @@ constexpr ::capnp::Kind QueryReader::_capnpPrivate::kind; constexpr ::capnp::_::RawSchema const* QueryReader::_capnpPrivate::schema; #endif // !CAPNP_LITE +// Delete +constexpr uint16_t Delete::_capnpPrivate::dataWordSize; +constexpr uint16_t Delete::_capnpPrivate::pointerCount; +#if !CAPNP_LITE +constexpr ::capnp::Kind Delete::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Delete::_capnpPrivate::schema; +#endif // !CAPNP_LITE + // ResultCellSlab constexpr uint16_t ResultCellSlab::_capnpPrivate::dataWordSize; constexpr uint16_t ResultCellSlab::_capnpPrivate::pointerCount; diff --git a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h index 037535b5d7a..3a5f937bbfc 100644 --- a/tiledb/sm/serialization/posix/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/posix/tiledb-rest.capnp.h @@ -53,6 +53,7 @@ CAPNP_DECLARE_SCHEMA(dac6a7f675c57409); CAPNP_DECLARE_SCHEMA(afc739d5c01e6496); CAPNP_DECLARE_SCHEMA(eaf57cb9871fc06f); CAPNP_DECLARE_SCHEMA(e19754f813ccf79c); +CAPNP_DECLARE_SCHEMA(d74f5fed155d316c); CAPNP_DECLARE_SCHEMA(def87cead82188e7); CAPNP_DECLARE_SCHEMA(c1a2d010de779de5); CAPNP_DECLARE_SCHEMA(c86c77b5f6a2bf0f); @@ -793,6 +794,23 @@ struct QueryReader { }; }; +struct Delete { + Delete() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(d74f5fed155d316c, 0, 2) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + struct ResultCellSlab { ResultCellSlab() = delete; @@ -869,7 +887,7 @@ struct Query { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(96ba49d0f8b23ccc, 4, 12) + CAPNP_DECLARE_STRUCT_HEADER(96ba49d0f8b23ccc, 4, 13) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -6622,6 +6640,122 @@ class QueryReader::Pipeline { }; #endif // !CAPNP_LITE +class Delete::Reader { + public: + typedef Delete Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasCondition() const; + inline ::tiledb::sm::serialization::capnp::Condition::Reader getCondition() + const; + + inline bool hasStats() const; + inline ::tiledb::sm::serialization::capnp::Stats::Reader getStats() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Delete::Builder { + public: + typedef Delete Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasCondition(); + inline ::tiledb::sm::serialization::capnp::Condition::Builder getCondition(); + inline void setCondition( + ::tiledb::sm::serialization::capnp::Condition::Reader value); + inline ::tiledb::sm::serialization::capnp::Condition::Builder initCondition(); + inline void adoptCondition( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition> + disownCondition(); + + inline bool hasStats(); + inline ::tiledb::sm::serialization::capnp::Stats::Builder getStats(); + inline void setStats(::tiledb::sm::serialization::capnp::Stats::Reader value); + inline ::tiledb::sm::serialization::capnp::Stats::Builder initStats(); + inline void adoptStats( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats> + disownStats(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Delete::Pipeline { + public: + typedef Delete Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + inline ::tiledb::sm::serialization::capnp::Condition::Pipeline getCondition(); + inline ::tiledb::sm::serialization::capnp::Stats::Pipeline getStats(); + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class ResultCellSlab::Reader { public: typedef ResultCellSlab Reads; @@ -7204,6 +7338,9 @@ class Query::Reader { inline ::tiledb::sm::serialization::capnp::QueryReader::Reader getDenseReader() const; + inline bool hasDelete() const; + inline ::tiledb::sm::serialization::capnp::Delete::Reader getDelete() const; + private: ::capnp::_::StructReader _reader; template @@ -7380,6 +7517,16 @@ class Query::Builder { inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::QueryReader> disownDenseReader(); + inline bool hasDelete(); + inline ::tiledb::sm::serialization::capnp::Delete::Builder getDelete(); + inline void setDelete( + ::tiledb::sm::serialization::capnp::Delete::Reader value); + inline ::tiledb::sm::serialization::capnp::Delete::Builder initDelete(); + inline void adoptDelete( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete> + disownDelete(); + private: ::capnp::_::StructBuilder _builder; template @@ -7410,6 +7557,7 @@ class Query::Pipeline { getReaderIndex(); inline ::tiledb::sm::serialization::capnp::QueryReader::Pipeline getDenseReader(); + inline ::tiledb::sm::serialization::capnp::Delete::Pipeline getDelete(); private: ::capnp::AnyPointer::Pipeline _typeless; @@ -16118,6 +16266,108 @@ QueryReader::Builder::disownStats() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool Delete::Reader::hasCondition() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Delete::Builder::hasCondition() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Condition::Reader +Delete::Reader::getCondition() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Condition::Builder +Delete::Builder::getCondition() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Condition::Pipeline +Delete::Pipeline::getCondition() { + return ::tiledb::sm::serialization::capnp::Condition::Pipeline( + _typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void Delete::Builder::setCondition( + ::tiledb::sm::serialization::capnp::Condition::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Condition>:: + set(_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Condition::Builder +Delete::Builder::initCondition() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Delete::Builder::adoptCondition( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Condition>:: + adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition> +Delete::Builder::disownCondition() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Delete::Reader::hasStats() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Delete::Builder::hasStats() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Stats::Reader +Delete::Reader::getStats() const { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + get(_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Stats::Builder +Delete::Builder::getStats() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + get(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Stats::Pipeline +Delete::Pipeline::getStats() { + return ::tiledb::sm::serialization::capnp::Stats::Pipeline( + _typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void Delete::Builder::setStats( + ::tiledb::sm::serialization::capnp::Stats::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Stats::Builder +Delete::Builder::initStats() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + init(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Delete::Builder::adoptStats( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats> +Delete::Builder::disownStats() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + inline ::uint32_t ResultCellSlab::Reader::getFragIdx() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -17263,6 +17513,58 @@ Query::Builder::disownDenseReader() { _builder.getPointerField(::capnp::bounded<11>() * ::capnp::POINTERS)); } +inline bool Query::Reader::hasDelete() const { + return !_reader.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Query::Builder::hasDelete() { + return !_builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Delete::Reader +Query::Reader::getDelete() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::get( + _reader.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Delete::Builder +Query::Builder::getDelete() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::get( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Delete::Pipeline +Query::Pipeline::getDelete() { + return ::tiledb::sm::serialization::capnp::Delete::Pipeline( + _typeless.getPointerField(12)); +} +#endif // !CAPNP_LITE +inline void Query::Builder::setDelete( + ::tiledb::sm::serialization::capnp::Delete::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::set( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Delete::Builder +Query::Builder::initDelete() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::init( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} +inline void Query::Builder::adoptDelete( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::adopt( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete> +Query::Builder::disownDelete() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::disown( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} + inline bool NonEmptyDomain::Reader::hasNonEmptyDomain() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); diff --git a/tiledb/sm/serialization/query.cc b/tiledb/sm/serialization/query.cc index 49a0369c8e4..ee652fcc81d 100644 --- a/tiledb/sm/serialization/query.cc +++ b/tiledb/sm/serialization/query.cc @@ -60,6 +60,7 @@ #include "tiledb/sm/fragment/fragment_metadata.h" #include "tiledb/sm/misc/hash.h" #include "tiledb/sm/misc/parse_argument.h" +#include "tiledb/sm/query/deletes_and_updates/deletes.h" #include "tiledb/sm/query/readers/dense_reader.h" #include "tiledb/sm/query/legacy/reader.h" #include "tiledb/sm/query/readers/sparse_global_order_reader.h" @@ -1100,6 +1101,50 @@ Status dense_reader_from_capnp( return Status::Ok(); } +Status delete_from_capnp( + const capnp::Delete::Reader& delete_reader, + Query* query, + Deletes* delete_strategy) { + // Query condition + if (delete_reader.hasCondition()) { + auto condition_reader = delete_reader.getCondition(); + QueryCondition condition; + RETURN_NOT_OK(condition_from_capnp(condition_reader, &condition)); + RETURN_NOT_OK(query->set_condition(condition)); + } + + // If cap'n proto object has stats set it on c++ object + if (delete_reader.hasStats()) { + stats::Stats* stats = delete_strategy->stats(); + // We should always have a stats here + if (stats != nullptr) { + RETURN_NOT_OK(stats_from_capnp(delete_reader.getStats(), stats)); + } + } + + return Status::Ok(); +} + +Status delete_to_capnp( + const Query& query, + Deletes& delete_strategy, + capnp::Delete::Builder* delete_builder) { + const QueryCondition* condition = query.condition(); + if (!condition->empty()) { + auto condition_builder = delete_builder->initCondition(); + RETURN_NOT_OK(condition_to_capnp(*condition, &condition_builder)); + } + + // If stats object exists set its cap'n proto object + stats::Stats* stats = delete_strategy.stats(); + if (stats != nullptr) { + auto stats_builder = delete_builder->initStats(); + RETURN_NOT_OK(stats_to_capnp(*stats, &stats_builder)); + } + + return Status::Ok(); +} + Status writer_to_capnp( const Query& query, WriterBase& writer, @@ -1164,7 +1209,8 @@ Status query_to_capnp( auto array = query.array(); auto query_type = query.type(); - if (query_type != QueryType::READ && query_type != QueryType::WRITE) { + if (query_type != QueryType::READ && query_type != QueryType::WRITE && + query_type != QueryType::DELETE) { return LOG_STATUS( Status_SerializationError("Cannot serialize; Unsupported query type.")); } @@ -1273,7 +1319,7 @@ Status query_to_capnp( auto reader = dynamic_cast(query.strategy()); RETURN_NOT_OK(reader_to_capnp(query, *reader, &builder)); } - } else { + } else if (type == QueryType::WRITE) { auto builder = query_builder->initWriter(); auto writer = dynamic_cast(query.strategy(true)); @@ -1282,6 +1328,10 @@ Status query_to_capnp( writer->offsets_extra_element()); query_builder->setVarOffsetsBitsize(writer->offsets_bitsize()); RETURN_NOT_OK(writer_to_capnp(query, *writer, &builder)); + } else { + auto builder = query_builder->initDelete(); + auto delete_strategy = dynamic_cast(query.strategy(true)); + RETURN_NOT_OK(delete_to_capnp(query, *delete_strategy, &builder)); } // Serialize Config @@ -1326,7 +1376,8 @@ Status query_from_capnp( " but got serialized type for " + query_reader.getType().cStr())); } - if (query_type != QueryType::READ && query_type != QueryType::WRITE) { + if (query_type != QueryType::READ && query_type != QueryType::WRITE && + query_type != QueryType::DELETE) { return LOG_STATUS(Status_SerializationError( "Cannot deserialize; Unsupported query type.")); } @@ -1390,7 +1441,7 @@ Status query_from_capnp( if (type == QueryType::READ && context == SerializationContext::SERVER) { const QueryBuffer& query_buffer = query->buffer(name); // We use the query_buffer directly in order to get the original buffer - // sizes This avoid a problem where an incomplete query will change the + // sizes. This avoid a problem where an incomplete query will change the // users buffer size to the smaller results and we end up not being able // to correctly calculate if the new results can fit into the users buffer if (var_size) { @@ -1420,7 +1471,7 @@ Status query_from_capnp( query_buffer.original_validity_vector_size_; } } - } else { + } else if (query_type == QueryType::WRITE || type == QueryType::READ) { // For writes we need to use get_buffer and clientside if (var_size) { if (!nullable) { @@ -1475,6 +1526,8 @@ Status query_from_capnp( existing_validity_buffer_size = *existing_validity_buffer_size_ptr; } } + } else { + // Delete queries have nothing to deserialize. } if (context == SerializationContext::CLIENT) { @@ -1700,7 +1753,7 @@ Status query_from_capnp( false)); } } - } else { + } else if (query_type == QueryType::WRITE) { // On writes, just set buffer pointers wrapping the data in the message. if (var_size) { auto* offsets = reinterpret_cast(attribute_buffer_start); @@ -1771,6 +1824,8 @@ Status query_from_capnp( &attr_state->validity_len_size)); } } + } else { + // Delete queries have no buffers to deserialize. } } } @@ -1816,7 +1871,7 @@ Status query_from_capnp( dynamic_cast(query->strategy()), compute_tp)); } - } else { + } else if (query_type == QueryType::WRITE) { auto writer_reader = query_reader.getWriter(); auto writer = dynamic_cast(query->strategy()); @@ -1855,6 +1910,10 @@ Status query_from_capnp( RETURN_NOT_OK(query->set_subarray_unsafe(subarray)); } } + } else { + auto delete_reader = query_reader.getDelete(); + auto delete_strategy = dynamic_cast(query->strategy()); + RETURN_NOT_OK(delete_from_capnp(delete_reader, query, delete_strategy)); } // Deserialize status. This must come last because various setters above diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index 30c8a29d086..d9e85aa5b16 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -485,6 +485,16 @@ struct QueryReader { # Stats object } +struct Delete { + # Delete struct + + condition @0 :Condition; + # The delete condition + + stats @1 :Stats; + # Stats object +} + struct ResultCellSlab { # Result cell slab @@ -592,6 +602,9 @@ struct Query { denseReader @16 :QueryReader; # denseReader contains data needed for continuation of incomplete dense reads with dense reader + + delete @17 :Delete; + #delete constains data for delete queries } struct NonEmptyDomain { diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ index 3b1451962e7..bdb4589c90b 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.c++ @@ -3966,6 +3966,70 @@ const ::capnp::_::RawSchema s_e19754f813ccf79c = { 4, 5, i_e19754f813ccf79c, nullptr, nullptr, { &s_e19754f813ccf79c, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE +static const ::capnp::_::AlignedData<49> b_d74f5fed155d316c = { + { 0, 0, 0, 0, 5, 0, 6, 0, + 108, 49, 93, 21, 237, 95, 79, 215, + 18, 0, 0, 0, 1, 0, 0, 0, + 127, 216, 135, 181, 36, 146, 125, 181, + 2, 0, 7, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 21, 0, 0, 0, 202, 0, 0, 0, + 33, 0, 0, 0, 7, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 29, 0, 0, 0, 119, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 116, 105, 108, 101, 100, 98, 45, 114, + 101, 115, 116, 46, 99, 97, 112, 110, + 112, 58, 68, 101, 108, 101, 116, 101, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 1, 0, + 8, 0, 0, 0, 3, 0, 4, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 41, 0, 0, 0, 82, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 40, 0, 0, 0, 3, 0, 1, 0, + 52, 0, 0, 0, 2, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 49, 0, 0, 0, 50, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 44, 0, 0, 0, 3, 0, 1, 0, + 56, 0, 0, 0, 2, 0, 1, 0, + 99, 111, 110, 100, 105, 116, 105, 111, + 110, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 111, 192, 31, 135, 185, 124, 245, 234, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 115, 116, 97, 116, 115, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 164, 161, 6, 21, 161, 54, 224, 199, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, } +}; +::capnp::word const* const bp_d74f5fed155d316c = b_d74f5fed155d316c.words; +#if !CAPNP_LITE +static const ::capnp::_::RawSchema* const d_d74f5fed155d316c[] = { + &s_c7e036a11506a1a4, + &s_eaf57cb9871fc06f, +}; +static const uint16_t m_d74f5fed155d316c[] = {0, 1}; +static const uint16_t i_d74f5fed155d316c[] = {0, 1}; +const ::capnp::_::RawSchema s_d74f5fed155d316c = { + 0xd74f5fed155d316c, b_d74f5fed155d316c.words, 49, d_d74f5fed155d316c, m_d74f5fed155d316c, + 2, 2, i_d74f5fed155d316c, nullptr, nullptr, { &s_d74f5fed155d316c, nullptr, nullptr, 0, 0, nullptr } +}; +#endif // !CAPNP_LITE static const ::capnp::_::AlignedData<79> b_def87cead82188e7 = { { 0, 0, 0, 0, 5, 0, 6, 0, 231, 136, 33, 216, 234, 124, 248, 222, @@ -4319,143 +4383,150 @@ const ::capnp::_::RawSchema s_9b9a5fc7713a8692 = { 4, 5, i_9b9a5fc7713a8692, nullptr, nullptr, { &s_9b9a5fc7713a8692, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE -static const ::capnp::_::AlignedData<294> b_96ba49d0f8b23ccc = { +static const ::capnp::_::AlignedData<309> b_96ba49d0f8b23ccc = { { 0, 0, 0, 0, 5, 0, 6, 0, 204, 60, 178, 248, 208, 73, 186, 150, 18, 0, 0, 0, 1, 0, 4, 0, 127, 216, 135, 181, 36, 146, 125, 181, - 12, 0, 7, 0, 0, 0, 0, 0, + 13, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 194, 0, 0, 0, 29, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 25, 0, 0, 0, 191, 3, 0, 0, + 25, 0, 0, 0, 247, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 105, 108, 101, 100, 98, 45, 114, 101, 115, 116, 46, 99, 97, 112, 110, 112, 58, 81, 117, 101, 114, 121, 0, 0, 0, 0, 0, 1, 0, 1, 0, - 68, 0, 0, 0, 3, 0, 4, 0, + 72, 0, 0, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 205, 1, 0, 0, 186, 0, 0, 0, + 233, 1, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 208, 1, 0, 0, 3, 0, 1, 0, - 236, 1, 0, 0, 2, 0, 1, 0, + 236, 1, 0, 0, 3, 0, 1, 0, + 8, 2, 0, 0, 2, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 233, 1, 0, 0, 58, 0, 0, 0, + 5, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 228, 1, 0, 0, 3, 0, 1, 0, - 240, 1, 0, 0, 2, 0, 1, 0, + 0, 2, 0, 0, 3, 0, 1, 0, + 12, 2, 0, 0, 2, 0, 1, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 237, 1, 0, 0, 58, 0, 0, 0, + 9, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 232, 1, 0, 0, 3, 0, 1, 0, - 244, 1, 0, 0, 2, 0, 1, 0, + 4, 2, 0, 0, 3, 0, 1, 0, + 16, 2, 0, 0, 2, 0, 1, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 241, 1, 0, 0, 42, 0, 0, 0, + 13, 2, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 236, 1, 0, 0, 3, 0, 1, 0, - 248, 1, 0, 0, 2, 0, 1, 0, + 8, 2, 0, 0, 3, 0, 1, 0, + 20, 2, 0, 0, 2, 0, 1, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 245, 1, 0, 0, 58, 0, 0, 0, + 17, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 240, 1, 0, 0, 3, 0, 1, 0, - 252, 1, 0, 0, 2, 0, 1, 0, + 12, 2, 0, 0, 3, 0, 1, 0, + 24, 2, 0, 0, 2, 0, 1, 0, 5, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 249, 1, 0, 0, 58, 0, 0, 0, + 21, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 244, 1, 0, 0, 3, 0, 1, 0, - 0, 2, 0, 0, 2, 0, 1, 0, + 16, 2, 0, 0, 3, 0, 1, 0, + 28, 2, 0, 0, 2, 0, 1, 0, 6, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 253, 1, 0, 0, 50, 0, 0, 0, + 25, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 248, 1, 0, 0, 3, 0, 1, 0, - 4, 2, 0, 0, 2, 0, 1, 0, + 20, 2, 0, 0, 3, 0, 1, 0, + 32, 2, 0, 0, 2, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 2, 0, 0, 226, 0, 0, 0, + 29, 2, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 8, 2, 0, 0, 3, 0, 1, 0, - 20, 2, 0, 0, 2, 0, 1, 0, + 36, 2, 0, 0, 3, 0, 1, 0, + 48, 2, 0, 0, 2, 0, 1, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 17, 2, 0, 0, 186, 0, 0, 0, + 45, 2, 0, 0, 186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 20, 2, 0, 0, 3, 0, 1, 0, - 32, 2, 0, 0, 2, 0, 1, 0, + 48, 2, 0, 0, 3, 0, 1, 0, + 60, 2, 0, 0, 2, 0, 1, 0, 9, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 29, 2, 0, 0, 202, 0, 0, 0, + 57, 2, 0, 0, 202, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 36, 2, 0, 0, 3, 0, 1, 0, - 48, 2, 0, 0, 2, 0, 1, 0, + 64, 2, 0, 0, 3, 0, 1, 0, + 76, 2, 0, 0, 2, 0, 1, 0, 10, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 45, 2, 0, 0, 122, 0, 0, 0, + 73, 2, 0, 0, 122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 44, 2, 0, 0, 3, 0, 1, 0, - 56, 2, 0, 0, 2, 0, 1, 0, + 72, 2, 0, 0, 3, 0, 1, 0, + 84, 2, 0, 0, 2, 0, 1, 0, 11, 0, 0, 0, 192, 0, 0, 0, 0, 0, 1, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 53, 2, 0, 0, 210, 0, 0, 0, + 81, 2, 0, 0, 210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 60, 2, 0, 0, 3, 0, 1, 0, - 72, 2, 0, 0, 2, 0, 1, 0, + 88, 2, 0, 0, 3, 0, 1, 0, + 100, 2, 0, 0, 2, 0, 1, 0, 12, 0, 0, 0, 7, 0, 0, 0, 0, 0, 1, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 69, 2, 0, 0, 146, 0, 0, 0, + 97, 2, 0, 0, 146, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 72, 2, 0, 0, 3, 0, 1, 0, - 84, 2, 0, 0, 2, 0, 1, 0, + 100, 2, 0, 0, 3, 0, 1, 0, + 112, 2, 0, 0, 2, 0, 1, 0, 13, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 81, 2, 0, 0, 58, 0, 0, 0, + 109, 2, 0, 0, 58, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 76, 2, 0, 0, 3, 0, 1, 0, - 88, 2, 0, 0, 2, 0, 1, 0, + 104, 2, 0, 0, 3, 0, 1, 0, + 116, 2, 0, 0, 2, 0, 1, 0, 14, 0, 0, 0, 9, 0, 0, 0, 0, 0, 1, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 85, 2, 0, 0, 50, 0, 0, 0, + 113, 2, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 80, 2, 0, 0, 3, 0, 1, 0, - 92, 2, 0, 0, 2, 0, 1, 0, + 108, 2, 0, 0, 3, 0, 1, 0, + 120, 2, 0, 0, 2, 0, 1, 0, 15, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 89, 2, 0, 0, 98, 0, 0, 0, + 117, 2, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 88, 2, 0, 0, 3, 0, 1, 0, - 100, 2, 0, 0, 2, 0, 1, 0, + 116, 2, 0, 0, 3, 0, 1, 0, + 128, 2, 0, 0, 2, 0, 1, 0, 16, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 97, 2, 0, 0, 98, 0, 0, 0, + 125, 2, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 96, 2, 0, 0, 3, 0, 1, 0, - 108, 2, 0, 0, 2, 0, 1, 0, + 124, 2, 0, 0, 3, 0, 1, 0, + 136, 2, 0, 0, 2, 0, 1, 0, + 17, 0, 0, 0, 12, 0, 0, 0, + 0, 0, 1, 0, 17, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 133, 2, 0, 0, 58, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 128, 2, 0, 0, 3, 0, 1, 0, + 140, 2, 0, 0, 2, 0, 1, 0, 97, 116, 116, 114, 105, 98, 117, 116, 101, 66, 117, 102, 102, 101, 114, 72, 101, 97, 100, 101, 114, 115, 0, 0, @@ -4611,6 +4682,14 @@ static const ::capnp::_::AlignedData<294> b_96ba49d0f8b23ccc = { 156, 247, 204, 19, 248, 84, 151, 225, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 100, 101, 108, 101, 116, 101, 0, 0, + 16, 0, 0, 0, 0, 0, 0, 0, + 108, 49, 93, 21, 237, 95, 79, 215, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, } @@ -4624,13 +4703,14 @@ static const ::capnp::_::RawSchema* const d_96ba49d0f8b23ccc[] = { &s_b6c95b4b8111ad36, &s_c7e036a11506a1a4, &s_d20a578112fa92a2, + &s_d74f5fed155d316c, &s_e19754f813ccf79c, }; -static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 13, 16, 1, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4}; -static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}; +static const uint16_t m_96ba49d0f8b23ccc[] = {6, 0, 13, 17, 16, 1, 5, 15, 14, 2, 7, 9, 8, 3, 11, 12, 10, 4}; +static const uint16_t i_96ba49d0f8b23ccc[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}; const ::capnp::_::RawSchema s_96ba49d0f8b23ccc = { - 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 294, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, - 7, 17, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr } + 0x96ba49d0f8b23ccc, b_96ba49d0f8b23ccc.words, 309, d_96ba49d0f8b23ccc, m_96ba49d0f8b23ccc, + 8, 18, i_96ba49d0f8b23ccc, nullptr, nullptr, { &s_96ba49d0f8b23ccc, nullptr, nullptr, 0, 0, nullptr } }; #endif // !CAPNP_LITE static const ::capnp::_::AlignedData<69> b_9df6f2a42c4e5f0b = { @@ -6254,6 +6334,14 @@ constexpr ::capnp::Kind QueryReader::_capnpPrivate::kind; constexpr ::capnp::_::RawSchema const* QueryReader::_capnpPrivate::schema; #endif // !CAPNP_LITE +// Delete +constexpr uint16_t Delete::_capnpPrivate::dataWordSize; +constexpr uint16_t Delete::_capnpPrivate::pointerCount; +#if !CAPNP_LITE +constexpr ::capnp::Kind Delete::_capnpPrivate::kind; +constexpr ::capnp::_::RawSchema const* Delete::_capnpPrivate::schema; +#endif // !CAPNP_LITE + // ResultCellSlab constexpr uint16_t ResultCellSlab::_capnpPrivate::dataWordSize; constexpr uint16_t ResultCellSlab::_capnpPrivate::pointerCount; diff --git a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h index 037535b5d7a..3a5f937bbfc 100644 --- a/tiledb/sm/serialization/win32/tiledb-rest.capnp.h +++ b/tiledb/sm/serialization/win32/tiledb-rest.capnp.h @@ -53,6 +53,7 @@ CAPNP_DECLARE_SCHEMA(dac6a7f675c57409); CAPNP_DECLARE_SCHEMA(afc739d5c01e6496); CAPNP_DECLARE_SCHEMA(eaf57cb9871fc06f); CAPNP_DECLARE_SCHEMA(e19754f813ccf79c); +CAPNP_DECLARE_SCHEMA(d74f5fed155d316c); CAPNP_DECLARE_SCHEMA(def87cead82188e7); CAPNP_DECLARE_SCHEMA(c1a2d010de779de5); CAPNP_DECLARE_SCHEMA(c86c77b5f6a2bf0f); @@ -793,6 +794,23 @@ struct QueryReader { }; }; +struct Delete { + Delete() = delete; + + class Reader; + class Builder; + class Pipeline; + + struct _capnpPrivate { + CAPNP_DECLARE_STRUCT_HEADER(d74f5fed155d316c, 0, 2) +#if !CAPNP_LITE + static constexpr ::capnp::_::RawBrandedSchema const* brand() { + return &schema->defaultBrand; + } +#endif // !CAPNP_LITE + }; +}; + struct ResultCellSlab { ResultCellSlab() = delete; @@ -869,7 +887,7 @@ struct Query { class Pipeline; struct _capnpPrivate { - CAPNP_DECLARE_STRUCT_HEADER(96ba49d0f8b23ccc, 4, 12) + CAPNP_DECLARE_STRUCT_HEADER(96ba49d0f8b23ccc, 4, 13) #if !CAPNP_LITE static constexpr ::capnp::_::RawBrandedSchema const* brand() { return &schema->defaultBrand; @@ -6622,6 +6640,122 @@ class QueryReader::Pipeline { }; #endif // !CAPNP_LITE +class Delete::Reader { + public: + typedef Delete Reads; + + Reader() = default; + inline explicit Reader(::capnp::_::StructReader base) + : _reader(base) { + } + + inline ::capnp::MessageSize totalSize() const { + return _reader.totalSize().asPublic(); + } + +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return ::capnp::_::structString(_reader, *_capnpPrivate::brand()); + } +#endif // !CAPNP_LITE + + inline bool hasCondition() const; + inline ::tiledb::sm::serialization::capnp::Condition::Reader getCondition() + const; + + inline bool hasStats() const; + inline ::tiledb::sm::serialization::capnp::Stats::Reader getStats() const; + + private: + ::capnp::_::StructReader _reader; + template + friend struct ::capnp::ToDynamic_; + template + friend struct ::capnp::_::PointerHelpers; + template + friend struct ::capnp::List; + friend class ::capnp::MessageBuilder; + friend class ::capnp::Orphanage; +}; + +class Delete::Builder { + public: + typedef Delete Builds; + + Builder() = delete; // Deleted to discourage incorrect usage. + // You can explicitly initialize to nullptr instead. + inline Builder(decltype(nullptr)) { + } + inline explicit Builder(::capnp::_::StructBuilder base) + : _builder(base) { + } + inline operator Reader() const { + return Reader(_builder.asReader()); + } + inline Reader asReader() const { + return *this; + } + + inline ::capnp::MessageSize totalSize() const { + return asReader().totalSize(); + } +#if !CAPNP_LITE + inline ::kj::StringTree toString() const { + return asReader().toString(); + } +#endif // !CAPNP_LITE + + inline bool hasCondition(); + inline ::tiledb::sm::serialization::capnp::Condition::Builder getCondition(); + inline void setCondition( + ::tiledb::sm::serialization::capnp::Condition::Reader value); + inline ::tiledb::sm::serialization::capnp::Condition::Builder initCondition(); + inline void adoptCondition( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition> + disownCondition(); + + inline bool hasStats(); + inline ::tiledb::sm::serialization::capnp::Stats::Builder getStats(); + inline void setStats(::tiledb::sm::serialization::capnp::Stats::Reader value); + inline ::tiledb::sm::serialization::capnp::Stats::Builder initStats(); + inline void adoptStats( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats> + disownStats(); + + private: + ::capnp::_::StructBuilder _builder; + template + friend struct ::capnp::ToDynamic_; + friend class ::capnp::Orphanage; + template + friend struct ::capnp::_::PointerHelpers; +}; + +#if !CAPNP_LITE +class Delete::Pipeline { + public: + typedef Delete Pipelines; + + inline Pipeline(decltype(nullptr)) + : _typeless(nullptr) { + } + inline explicit Pipeline(::capnp::AnyPointer::Pipeline&& typeless) + : _typeless(kj::mv(typeless)) { + } + + inline ::tiledb::sm::serialization::capnp::Condition::Pipeline getCondition(); + inline ::tiledb::sm::serialization::capnp::Stats::Pipeline getStats(); + + private: + ::capnp::AnyPointer::Pipeline _typeless; + friend class ::capnp::PipelineHook; + template + friend struct ::capnp::ToDynamic_; +}; +#endif // !CAPNP_LITE + class ResultCellSlab::Reader { public: typedef ResultCellSlab Reads; @@ -7204,6 +7338,9 @@ class Query::Reader { inline ::tiledb::sm::serialization::capnp::QueryReader::Reader getDenseReader() const; + inline bool hasDelete() const; + inline ::tiledb::sm::serialization::capnp::Delete::Reader getDelete() const; + private: ::capnp::_::StructReader _reader; template @@ -7380,6 +7517,16 @@ class Query::Builder { inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::QueryReader> disownDenseReader(); + inline bool hasDelete(); + inline ::tiledb::sm::serialization::capnp::Delete::Builder getDelete(); + inline void setDelete( + ::tiledb::sm::serialization::capnp::Delete::Reader value); + inline ::tiledb::sm::serialization::capnp::Delete::Builder initDelete(); + inline void adoptDelete( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete>&& value); + inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete> + disownDelete(); + private: ::capnp::_::StructBuilder _builder; template @@ -7410,6 +7557,7 @@ class Query::Pipeline { getReaderIndex(); inline ::tiledb::sm::serialization::capnp::QueryReader::Pipeline getDenseReader(); + inline ::tiledb::sm::serialization::capnp::Delete::Pipeline getDelete(); private: ::capnp::AnyPointer::Pipeline _typeless; @@ -16118,6 +16266,108 @@ QueryReader::Builder::disownStats() { _builder.getPointerField(::capnp::bounded<4>() * ::capnp::POINTERS)); } +inline bool Delete::Reader::hasCondition() const { + return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Delete::Builder::hasCondition() { + return !_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Condition::Reader +Delete::Reader::getCondition() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::get( + _reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Condition::Builder +Delete::Builder::getCondition() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::get( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Condition::Pipeline +Delete::Pipeline::getCondition() { + return ::tiledb::sm::serialization::capnp::Condition::Pipeline( + _typeless.getPointerField(0)); +} +#endif // !CAPNP_LITE +inline void Delete::Builder::setCondition( + ::tiledb::sm::serialization::capnp::Condition::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Condition>:: + set(_builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Condition::Builder +Delete::Builder::initCondition() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::init( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} +inline void Delete::Builder::adoptCondition( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Condition>:: + adopt( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Condition> +Delete::Builder::disownCondition() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Condition>::disown( + _builder.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS)); +} + +inline bool Delete::Reader::hasStats() const { + return !_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Delete::Builder::hasStats() { + return !_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Stats::Reader +Delete::Reader::getStats() const { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + get(_reader.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Stats::Builder +Delete::Builder::getStats() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + get(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Stats::Pipeline +Delete::Pipeline::getStats() { + return ::tiledb::sm::serialization::capnp::Stats::Pipeline( + _typeless.getPointerField(1)); +} +#endif // !CAPNP_LITE +inline void Delete::Builder::setStats( + ::tiledb::sm::serialization::capnp::Stats::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>::set( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Stats::Builder +Delete::Builder::initStats() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + init(_builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} +inline void Delete::Builder::adoptStats( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>::adopt( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Stats> +Delete::Builder::disownStats() { + return ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Stats>:: + disown( + _builder.getPointerField(::capnp::bounded<1>() * ::capnp::POINTERS)); +} + inline ::uint32_t ResultCellSlab::Reader::getFragIdx() const { return _reader.getDataField<::uint32_t>( ::capnp::bounded<0>() * ::capnp::ELEMENTS); @@ -17263,6 +17513,58 @@ Query::Builder::disownDenseReader() { _builder.getPointerField(::capnp::bounded<11>() * ::capnp::POINTERS)); } +inline bool Query::Reader::hasDelete() const { + return !_reader.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS) + .isNull(); +} +inline bool Query::Builder::hasDelete() { + return !_builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS) + .isNull(); +} +inline ::tiledb::sm::serialization::capnp::Delete::Reader +Query::Reader::getDelete() const { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::get( + _reader.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} +inline ::tiledb::sm::serialization::capnp::Delete::Builder +Query::Builder::getDelete() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::get( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} +#if !CAPNP_LITE +inline ::tiledb::sm::serialization::capnp::Delete::Pipeline +Query::Pipeline::getDelete() { + return ::tiledb::sm::serialization::capnp::Delete::Pipeline( + _typeless.getPointerField(12)); +} +#endif // !CAPNP_LITE +inline void Query::Builder::setDelete( + ::tiledb::sm::serialization::capnp::Delete::Reader value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::set( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS), + value); +} +inline ::tiledb::sm::serialization::capnp::Delete::Builder +Query::Builder::initDelete() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::init( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} +inline void Query::Builder::adoptDelete( + ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete>&& value) { + ::capnp::_::PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::adopt( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS), + kj::mv(value)); +} +inline ::capnp::Orphan<::tiledb::sm::serialization::capnp::Delete> +Query::Builder::disownDelete() { + return ::capnp::_:: + PointerHelpers<::tiledb::sm::serialization::capnp::Delete>::disown( + _builder.getPointerField(::capnp::bounded<12>() * ::capnp::POINTERS)); +} + inline bool NonEmptyDomain::Reader::hasNonEmptyDomain() const { return !_reader.getPointerField(::capnp::bounded<0>() * ::capnp::POINTERS) .isNull(); From 044a6db749e8f18711f962fcc267694e8bb545db Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Tue, 16 Aug 2022 00:07:20 -0400 Subject: [PATCH 2/3] Add initial directory doc for delete folder --- tiledb/sm/query/DELETES_AND_UPDATES.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 tiledb/sm/query/DELETES_AND_UPDATES.md diff --git a/tiledb/sm/query/DELETES_AND_UPDATES.md b/tiledb/sm/query/DELETES_AND_UPDATES.md new file mode 100644 index 00000000000..94de7ecdfd9 --- /dev/null +++ b/tiledb/sm/query/DELETES_AND_UPDATES.md @@ -0,0 +1,4 @@ +# Deletes + +Deletes are implemented as a query type, `TILEDB_QUERY_TYPE_DELETE`, which requires +a QueryCondition to be set on the Query. Upon query submission, TileDB creates a [delete commit file](https://github.com/TileDB-Inc/TileDB/blob/dev/format_spec/delete_commit_file.md) containing a serialized version of the query condition. All attribute values matching the query condition expression are considered to be deleted as of the timestamp of the commit file. During reads, any delete commit files within the query range are loaded and deserialized, then the _negation_ of the stored query condition is applied to the final result. \ No newline at end of file From 3d4595e48fdaa59db661d8e67d61c9926a97e93f Mon Sep 17 00:00:00 2001 From: Isaiah Norton Date: Tue, 16 Aug 2022 00:09:30 -0400 Subject: [PATCH 3/3] Fix typo / clarify --- tiledb/sm/serialization/tiledb-rest.capnp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tiledb/sm/serialization/tiledb-rest.capnp b/tiledb/sm/serialization/tiledb-rest.capnp index d9e85aa5b16..82cc5ece454 100644 --- a/tiledb/sm/serialization/tiledb-rest.capnp +++ b/tiledb/sm/serialization/tiledb-rest.capnp @@ -117,7 +117,7 @@ struct ArraySchemaEvolution { # Attribute names to be dropped attributesToAdd @1 :List(Attribute); - # Attributes to be added + # Attributes to be added timestampRange @2 :List(UInt64); # Timestamp range of array schema @@ -447,7 +447,7 @@ struct ASTNode { # Expression node fields children @4 :List(ASTNode); - # A list of children + # A list of children combinationOp @5 :Text; # The combination logical operator @@ -604,7 +604,7 @@ struct Query { # denseReader contains data needed for continuation of incomplete dense reads with dense reader delete @17 :Delete; - #delete constains data for delete queries + # delete contains QueryCondition representing deletion expression } struct NonEmptyDomain { @@ -664,12 +664,12 @@ struct ArrayMetadata { struct EstimatedResultSize { # object representing estimated struct ResultSize { - # Result size + # Result size sizeFixed @0 :Float64; sizeVar @1 :Float64; sizeValidity @2 :Float64; } - + struct MemorySize { # Memory Size sizeFixed @0 :UInt64;