From d5ff15ad1e7a65145d122ba877b1fd59236af55c Mon Sep 17 00:00:00 2001 From: Luc Rancourt Date: Tue, 28 Jun 2022 21:57:02 +0300 Subject: [PATCH 1/3] Storage manager: exposing methods to load/store generic tile. This changes every paths in the storage manager to load/store generic tile to go through a common API that can also be used externally from the storage manager. --- TYPE: IMPROVEMENT DESC: Storage manager: exposing methods to load/store generic tile. --- tiledb/sm/storage_manager/storage_manager.cc | 278 +++++++------------ tiledb/sm/storage_manager/storage_manager.h | 25 ++ 2 files changed, 126 insertions(+), 177 deletions(-) diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 2814a078212..d6acd5c31fb 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1422,63 +1422,13 @@ StorageManager::load_array_schema_from_uri( const URI& schema_uri, const EncryptionKey& encryption_key) { auto timer_se = stats_->start_timer("sm_load_array_schema_from_uri"); - GenericTileIO tile_io(this, schema_uri); - Tile* tile = nullptr; - - // Get encryption key from config - if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { - bool found = false; - std::string encryption_key_from_cfg = - config_.get("sm.encryption_key", &found); - assert(found); - std::string encryption_type_from_cfg = - config_.get("sm.encryption_type", &found); - assert(found); - auto [st, etc] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK_TUPLE(st, nullopt); - EncryptionType encryption_type_cfg = etc.value(); - - EncryptionKey encryption_key_cfg; - if (encryption_key_from_cfg.empty()) { - RETURN_NOT_OK_TUPLE( - encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0), nullopt); - } else { - uint32_t key_length = 0; - if (EncryptionKey::is_valid_key_length( - encryption_type_cfg, - static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } - RETURN_NOT_OK_TUPLE( - encryption_key_cfg.set_key( - encryption_type_cfg, - (const void*)encryption_key_from_cfg.c_str(), - key_length), - nullopt); - } - RETURN_NOT_OK_TUPLE( - tile_io.read_generic(&tile, 0, encryption_key_cfg, config_), nullopt); - } else { - RETURN_NOT_OK_TUPLE( - tile_io.read_generic(&tile, 0, encryption_key, config_), nullopt); - } - - Buffer buff; - buff.realloc(tile->size()); - buff.set_size(tile->size()); - auto st = tile->read(buff.data(), 0, buff.size()); - tdb_delete(tile); + auto&& [st, buffer] = load_data_from_generic_tile(schema_uri, encryption_key); RETURN_NOT_OK_TUPLE(st, nullopt); - stats_->add_counter("read_array_schema_size", buff.size()); + stats_->add_counter("read_array_schema_size", buffer->size()); // Deserialize - ConstBuffer cbuff(&buff); + ConstBuffer cbuff(&*buffer); auto deserialized_schema{ArraySchema::deserialize(&cbuff, schema_uri)}; // #TODO Update catch statement after later StatusException changes @@ -1593,18 +1543,10 @@ Status StorageManager::load_array_metadata( metadata_buffs.resize(metadata_num); auto status = parallel_for(compute_tp_, 0, metadata_num, [&](size_t m) { const auto& uri = array_metadata_to_load[m].uri_; - GenericTileIO tile_io(this, uri); - auto tile = (Tile*)nullptr; - RETURN_NOT_OK(tile_io.read_generic(&tile, 0, encryption_key, config_)); - auto metadata_buff = make_shared(HERE()); - RETURN_NOT_OK(metadata_buff->realloc(tile->size())); - metadata_buff->set_size(tile->size()); - RETURN_NOT_OK_ELSE( - tile->read(metadata_buff->data(), 0, metadata_buff->size()), - tdb_delete(tile)); - tdb_delete(tile); - - metadata_buffs[m] = metadata_buff; + + auto&& [st, buffer] = load_data_from_generic_tile(uri, encryption_key); + RETURN_NOT_OK(st); + metadata_buffs[m] = make_shared(HERE(), std::move(*buffer)); return Status::Ok(); }); @@ -1892,22 +1834,10 @@ Status StorageManager::store_group_detail( if (!group_detail_dir_exists) RETURN_NOT_OK(create_dir(group_detail_folder_uri)); - // Write to file - Tile tile( - constants::generic_tile_datatype, - constants::generic_tile_cell_size, - 0, - buff.data(), - buff.size()); + RETURN_NOT_OK(store_data_to_generic_tile( + buff.data(), buff.size(), *group_detail_uri, encryption_key)); buff.disown_data(); - GenericTileIO tile_io(this, group_detail_uri.value()); - uint64_t nbytes; - st = tile_io.write_generic(&tile, encryption_key, &nbytes); - (void)nbytes; - if (st.ok()) - st = close_file(group_detail_uri.value()); - return st; } @@ -1937,23 +1867,11 @@ Status StorageManager::store_array_schema( if (!schema_dir_exists) RETURN_NOT_OK(create_dir(array_schema_dir_uri)); - // Write to file - Tile tile( - constants::generic_tile_datatype, - constants::generic_tile_cell_size, - 0, - buff.data(), - buff.size()); + RETURN_NOT_OK(store_data_to_generic_tile( + buff.data(), buff.size(), schema_uri, encryption_key)); buff.disown_data(); - GenericTileIO tile_io(this, schema_uri); - uint64_t nbytes; - Status st = tile_io.write_generic(&tile, encryption_key, &nbytes); - (void)nbytes; - if (st.ok()) - st = close_file(schema_uri); - - return st; + return Status::Ok(); } Status StorageManager::store_metadata( @@ -1978,26 +1896,39 @@ Status StorageManager::store_metadata( URI metadata_uri; RETURN_NOT_OK(metadata->get_uri(uri, &metadata_uri)); + RETURN_NOT_OK(store_data_to_generic_tile( + metadata_buff.data(), + metadata_buff.size(), + metadata_uri, + encryption_key)); + metadata_buff.disown_data(); + metadata_buff.clear(); + + return Status::Ok(); +} + +Status StorageManager::store_data_to_generic_tile( + void* data, + const size_t size, + const URI uri, + const EncryptionKey& encryption_key) { Tile tile( constants::generic_tile_datatype, constants::generic_tile_cell_size, 0, - metadata_buff.data(), - metadata_buff.size()); - metadata_buff.disown_data(); + data, + size); - GenericTileIO tile_io(this, metadata_uri); + GenericTileIO tile_io(this, uri); uint64_t nbytes; Status st = tile_io.write_generic(&tile, encryption_key, &nbytes); (void)nbytes; if (st.ok()) { - st = close_file(metadata_uri); + st = close_file(uri); } - metadata_buff.clear(); - - return st; + return Status::Ok(); } Status StorageManager::close_file(const URI& uri) { @@ -2072,63 +2003,13 @@ tuple>> StorageManager::load_group_from_uri( const URI& group_uri, const URI& uri, const EncryptionKey& encryption_key) { auto timer_se = stats_->start_timer("sm_load_group_from_uri"); - GenericTileIO tile_io(this, uri); - Tile* tile = nullptr; - - // Get encryption key from config - if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { - bool found = false; - std::string encryption_key_from_cfg = - config_.get("sm.encryption_key", &found); - assert(found); - std::string encryption_type_from_cfg = - config_.get("sm.encryption_type", &found); - assert(found); - auto [st, etc] = encryption_type_enum(encryption_type_from_cfg); - RETURN_NOT_OK_TUPLE(st, nullopt); - EncryptionType encryption_type_cfg = etc.value(); - - EncryptionKey encryption_key_cfg; - if (encryption_key_from_cfg.empty()) { - RETURN_NOT_OK_TUPLE( - encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0), nullopt); - } else { - uint32_t key_length = 0; - if (EncryptionKey::is_valid_key_length( - encryption_type_cfg, - static_cast(encryption_key_from_cfg.size()))) { - const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); - if (unit_test_cfg.array_encryption_key_length.is_set()) { - key_length = unit_test_cfg.array_encryption_key_length.get(); - } else { - key_length = static_cast(encryption_key_from_cfg.size()); - } - } - RETURN_NOT_OK_TUPLE( - encryption_key_cfg.set_key( - encryption_type_cfg, - (const void*)encryption_key_from_cfg.c_str(), - key_length), - nullopt); - } - RETURN_NOT_OK_TUPLE( - tile_io.read_generic(&tile, 0, encryption_key_cfg, config_), nullopt); - } else { - RETURN_NOT_OK_TUPLE( - tile_io.read_generic(&tile, 0, encryption_key, config_), nullopt); - } - - Buffer buff; - buff.realloc(tile->size()); - buff.set_size(tile->size()); - auto st = tile->read(buff.data(), 0, buff.size()); - tdb_delete(tile); + auto&& [st, buffer] = load_data_from_generic_tile(uri, encryption_key); RETURN_NOT_OK_TUPLE(st, nullopt); - stats_->add_counter("read_group_size", buff.size()); + stats_->add_counter("read_group_size", buffer->size()); // Deserialize - ConstBuffer cbuff(&buff); + ConstBuffer cbuff(&*buffer); return Group::deserialize(&cbuff, group_uri, this); } @@ -2208,18 +2089,10 @@ Status StorageManager::load_group_metadata( metadata_buffs.resize(metadata_num); auto status = parallel_for(compute_tp_, 0, metadata_num, [&](size_t m) { const auto& uri = group_metadata_to_load[m].uri_; - GenericTileIO tile_io(this, uri); - auto tile = (Tile*)nullptr; - RETURN_NOT_OK(tile_io.read_generic(&tile, 0, encryption_key, config_)); - auto metadata_buff = tdb::make_shared(HERE()); - RETURN_NOT_OK(metadata_buff->realloc(tile->size())); - metadata_buff->set_size(tile->size()); - RETURN_NOT_OK_ELSE( - tile->read(metadata_buff->data(), 0, metadata_buff->size()), - tdb_delete(tile)); - tdb_delete(tile); - - metadata_buffs[m] = metadata_buff; + + auto&& [st, buffer] = load_data_from_generic_tile(uri, encryption_key); + RETURN_NOT_OK(st); + metadata_buffs[m] = tdb::make_shared(HERE(), std::move(*buffer)); return Status::Ok(); }); @@ -2244,6 +2117,65 @@ Status StorageManager::load_group_metadata( return Status::Ok(); } +tuple> StorageManager::load_data_from_generic_tile( + const URI uri, const EncryptionKey& encryption_key) { + Tile* tile = nullptr; + GenericTileIO tile_io(this, uri); + + // Get encryption key from config + if (encryption_key.encryption_type() == EncryptionType::NO_ENCRYPTION) { + bool found = false; + std::string encryption_key_from_cfg = + config_.get("sm.encryption_key", &found); + assert(found); + std::string encryption_type_from_cfg = + config_.get("sm.encryption_type", &found); + assert(found); + auto [st, etc] = encryption_type_enum(encryption_type_from_cfg); + RETURN_NOT_OK_TUPLE(st, nullopt); + EncryptionType encryption_type_cfg = etc.value(); + + EncryptionKey encryption_key_cfg; + if (encryption_key_from_cfg.empty()) { + RETURN_NOT_OK_TUPLE( + encryption_key_cfg.set_key(encryption_type_cfg, nullptr, 0), nullopt); + } else { + uint32_t key_length = 0; + if (EncryptionKey::is_valid_key_length( + encryption_type_cfg, + static_cast(encryption_key_from_cfg.size()))) { + const UnitTestConfig& unit_test_cfg = UnitTestConfig::instance(); + if (unit_test_cfg.array_encryption_key_length.is_set()) { + key_length = unit_test_cfg.array_encryption_key_length.get(); + } else { + key_length = static_cast(encryption_key_from_cfg.size()); + } + } + RETURN_NOT_OK_TUPLE( + encryption_key_cfg.set_key( + encryption_type_cfg, + (const void*)encryption_key_from_cfg.c_str(), + key_length), + nullopt); + } + RETURN_NOT_OK_TUPLE( + tile_io.read_generic(&tile, 0, encryption_key_cfg, config_), nullopt); + } else { + RETURN_NOT_OK_TUPLE( + tile_io.read_generic(&tile, 0, encryption_key, config_), nullopt); + } + + Buffer buff; + buff.realloc(tile->size()); + buff.set_size(tile->size()); + auto st = tile->read(buff.data(), 0, buff.size()); + tdb_delete(tile); + + RETURN_NOT_OK_TUPLE(st, nullopt); + + return {Status::Ok(), buff}; +} + /* ****************************** */ /* PRIVATE METHODS */ /* ****************************** */ @@ -2336,16 +2268,8 @@ StorageManager::load_consolidated_fragment_meta( if (uri.to_string().empty()) return {Status::Ok(), nullopt}; - GenericTileIO tile_io(this, uri); - Tile* tile = nullptr; - RETURN_NOT_OK_TUPLE( - tile_io.read_generic(&tile, 0, enc_key, config_), nullopt); - - f_buff->realloc(tile->size()); - f_buff->set_size(tile->size()); - RETURN_NOT_OK_ELSE_TUPLE( - tile->read(f_buff->data(), 0, f_buff->size()), tdb_delete(tile), nullopt); - tdb_delete(tile); + auto&& [st, buffer] = load_data_from_generic_tile(uri, enc_key); + buffer->swap(*f_buff); stats_->add_counter("consolidated_frag_meta_size", f_buff->size()); diff --git a/tiledb/sm/storage_manager/storage_manager.h b/tiledb/sm/storage_manager/storage_manager.h index a75e6b7a19a..ad436bccb05 100644 --- a/tiledb/sm/storage_manager/storage_manager.h +++ b/tiledb/sm/storage_manager/storage_manager.h @@ -174,6 +174,16 @@ class StorageManager { const EncryptionKey& encryption_key, Metadata* metadata); + /** + * Load data from persistent storage. + * + * @param uri The object URI. + * @param encryption_key The encryption key to use. + * @return Status, Buffer with the data. + */ + tuple> load_data_from_generic_tile( + const URI uri, const EncryptionKey& encryption_key); + /** * Load a group detail from URI * @@ -940,6 +950,21 @@ class StorageManager { Status store_metadata( const URI& uri, const EncryptionKey& encryption_key, Metadata* metadata); + /** + * Stores data into persistent storage. + * + * @param data Data to store. + * @param size Size of the data. + * @param uri The object URI. + * @param encryption_key The encryption key to use. + * @return Status + */ + Status store_data_to_generic_tile( + void* data, + const size_t size, + const URI uri, + const EncryptionKey& encryption_key); + /** Closes a file, flushing its contents to persistent storage. */ Status close_file(const URI& uri); From e9fb3e6b25367ffbd13f151a769827f8d664f3c0 Mon Sep 17 00:00:00 2001 From: Luc Rancourt Date: Wed, 29 Jun 2022 15:19:22 +0300 Subject: [PATCH 2/3] Addressing feedback from @ypatia. --- tiledb/sm/fragment/fragment_metadata.cc | 23 ++-- tiledb/sm/storage_manager/storage_manager.cc | 54 ++++---- tiledb/sm/storage_manager/storage_manager.h | 4 +- tiledb/sm/tile/generic_tile_io.cc | 128 +++++++++++-------- tiledb/sm/tile/generic_tile_io.h | 16 +-- 5 files changed, 122 insertions(+), 103 deletions(-) diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index eb162b5a9b3..0ea519a2203 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -3561,15 +3561,15 @@ Status FragmentMetadata::load_v1_v2( std::string(constants::fragment_metadata_filename)); // Read metadata GenericTileIO tile_io(storage_manager_, fragment_metadata_uri); - auto tile = (Tile*)nullptr; - RETURN_NOT_OK(tile_io.read_generic( - &tile, 0, encryption_key, storage_manager_->config())); + auto&& [st, tile_opt] = + tile_io.read_generic(0, encryption_key, storage_manager_->config()); + RETURN_NOT_OK(st); + auto& tile = *tile_opt; Buffer buff; - RETURN_NOT_OK_ELSE(buff.realloc(tile->size()), tdb_delete(tile)); + RETURN_NOT_OK(buff.realloc(tile->size())); buff.set_size(tile->size()); - RETURN_NOT_OK_ELSE(tile->read(buff.data(), 0, buff.size()), tdb_delete(tile)); - tdb_delete(tile); + RETURN_NOT_OK(tile->read(buff.data(), 0, buff.size())); storage_manager_->stats()->add_counter("read_frag_meta_size", buff.size()); @@ -3982,15 +3982,14 @@ Status FragmentMetadata::read_generic_tile_from_file( // Read metadata GenericTileIO tile_io(storage_manager_, fragment_metadata_uri); - Tile* tile = nullptr; - RETURN_NOT_OK(tile_io.read_generic( - &tile, offset, encryption_key, storage_manager_->config())); + auto&& [st, tile_opt] = + tile_io.read_generic(offset, encryption_key, storage_manager_->config()); + RETURN_NOT_OK(st); + auto& tile = *tile_opt; buff->realloc(tile->size()); buff->set_size(tile->size()); - RETURN_NOT_OK_ELSE( - tile->read(buff->data(), 0, buff->size()), tdb_delete(tile)); - tdb_delete(tile); + RETURN_NOT_OK(tile->read(buff->data(), 0, buff->size())); return Status::Ok(); } diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index d6acd5c31fb..931c419925b 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -1140,17 +1140,18 @@ Status StorageManager::array_get_encryption( const ArrayDirectory& array_dir, EncryptionType* encryption_type) { const URI& uri = array_dir.uri(); - if (uri.is_invalid()) + if (uri.is_invalid()) { return logger_->status(Status_StorageManagerError( "Cannot get array encryption; Invalid array URI")); + } const URI& schema_uri = array_dir.latest_array_schema_uri(); // Read tile header - GenericTileIO::GenericTileHeader header; - RETURN_NOT_OK( - GenericTileIO::read_generic_tile_header(this, schema_uri, 0, &header)); - *encryption_type = static_cast(header.encryption_type); + auto&& [st, header] = + GenericTileIO::read_generic_tile_header(this, schema_uri, 0); + RETURN_NOT_OK(st); + *encryption_type = static_cast(header->encryption_type); return Status::Ok(); } @@ -1422,13 +1423,15 @@ StorageManager::load_array_schema_from_uri( const URI& schema_uri, const EncryptionKey& encryption_key) { auto timer_se = stats_->start_timer("sm_load_array_schema_from_uri"); - auto&& [st, buffer] = load_data_from_generic_tile(schema_uri, encryption_key); + auto&& [st, buffer_opt] = + load_data_from_generic_tile(schema_uri, encryption_key); RETURN_NOT_OK_TUPLE(st, nullopt); + auto& buffer = *buffer_opt; - stats_->add_counter("read_array_schema_size", buffer->size()); + stats_->add_counter("read_array_schema_size", buffer.size()); // Deserialize - ConstBuffer cbuff(&*buffer); + ConstBuffer cbuff(&buffer); auto deserialized_schema{ArraySchema::deserialize(&cbuff, schema_uri)}; // #TODO Update catch statement after later StatusException changes @@ -1902,7 +1905,6 @@ Status StorageManager::store_metadata( metadata_uri, encryption_key)); metadata_buff.disown_data(); - metadata_buff.clear(); return Status::Ok(); } @@ -1910,7 +1912,7 @@ Status StorageManager::store_metadata( Status StorageManager::store_data_to_generic_tile( void* data, const size_t size, - const URI uri, + const URI& uri, const EncryptionKey& encryption_key) { Tile tile( constants::generic_tile_datatype, @@ -1920,15 +1922,14 @@ Status StorageManager::store_data_to_generic_tile( size); GenericTileIO tile_io(this, uri); - uint64_t nbytes; + uint64_t nbytes = 0; Status st = tile_io.write_generic(&tile, encryption_key, &nbytes); - (void)nbytes; if (st.ok()) { st = close_file(uri); } - return Status::Ok(); + return st; } Status StorageManager::close_file(const URI& uri) { @@ -2003,13 +2004,14 @@ tuple>> StorageManager::load_group_from_uri( const URI& group_uri, const URI& uri, const EncryptionKey& encryption_key) { auto timer_se = stats_->start_timer("sm_load_group_from_uri"); - auto&& [st, buffer] = load_data_from_generic_tile(uri, encryption_key); + auto&& [st, buffer_opt] = load_data_from_generic_tile(uri, encryption_key); RETURN_NOT_OK_TUPLE(st, nullopt); + auto& buffer = *buffer_opt; - stats_->add_counter("read_group_size", buffer->size()); + stats_->add_counter("read_group_size", buffer.size()); // Deserialize - ConstBuffer cbuff(&*buffer); + ConstBuffer cbuff(&buffer); return Group::deserialize(&cbuff, group_uri, this); } @@ -2118,8 +2120,8 @@ Status StorageManager::load_group_metadata( } tuple> StorageManager::load_data_from_generic_tile( - const URI uri, const EncryptionKey& encryption_key) { - Tile* tile = nullptr; + const URI& uri, const EncryptionKey& encryption_key) { + tdb_unique_ptr tile; GenericTileIO tile_io(this, uri); // Get encryption key from config @@ -2158,18 +2160,21 @@ tuple> StorageManager::load_data_from_generic_tile( key_length), nullopt); } - RETURN_NOT_OK_TUPLE( - tile_io.read_generic(&tile, 0, encryption_key_cfg, config_), nullopt); + + auto&& [st1, tile_opt] = + tile_io.read_generic(0, encryption_key_cfg, config_); + RETURN_NOT_OK_TUPLE(st1, nullopt); + tile = std::move(*tile_opt); } else { - RETURN_NOT_OK_TUPLE( - tile_io.read_generic(&tile, 0, encryption_key, config_), nullopt); + auto&& [st1, tile_opt] = tile_io.read_generic(0, encryption_key, config_); + RETURN_NOT_OK_TUPLE(st1, nullopt); + tile = std::move(*tile_opt); } Buffer buff; - buff.realloc(tile->size()); + RETURN_NOT_OK_TUPLE(buff.realloc(tile->size()), std::nullopt); buff.set_size(tile->size()); auto st = tile->read(buff.data(), 0, buff.size()); - tdb_delete(tile); RETURN_NOT_OK_TUPLE(st, nullopt); @@ -2269,6 +2274,7 @@ StorageManager::load_consolidated_fragment_meta( return {Status::Ok(), nullopt}; auto&& [st, buffer] = load_data_from_generic_tile(uri, enc_key); + RETURN_NOT_OK_TUPLE(st, nullopt); buffer->swap(*f_buff); stats_->add_counter("consolidated_frag_meta_size", f_buff->size()); diff --git a/tiledb/sm/storage_manager/storage_manager.h b/tiledb/sm/storage_manager/storage_manager.h index ad436bccb05..68083f5defc 100644 --- a/tiledb/sm/storage_manager/storage_manager.h +++ b/tiledb/sm/storage_manager/storage_manager.h @@ -182,7 +182,7 @@ class StorageManager { * @return Status, Buffer with the data. */ tuple> load_data_from_generic_tile( - const URI uri, const EncryptionKey& encryption_key); + const URI& uri, const EncryptionKey& encryption_key); /** * Load a group detail from URI @@ -962,7 +962,7 @@ class StorageManager { Status store_data_to_generic_tile( void* data, const size_t size, - const URI uri, + const URI& uri, const EncryptionKey& encryption_key); /** Closes a file, flushing its contents to persistent storage. */ diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index 3b9c56d2d7e..91b050fcdf8 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -59,92 +59,112 @@ GenericTileIO::GenericTileIO(StorageManager* storage_manager, const URI& uri) /* API */ /* ****************************** */ -Status GenericTileIO::read_generic( - Tile** tile, +tuple>> GenericTileIO::read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config) { tdb_unique_ptr ret(tdb_new(Tile)); - GenericTileHeader header; - RETURN_NOT_OK( - read_generic_tile_header(storage_manager_, uri_, file_offset, &header)); + auto&& [st, header_opt] = + read_generic_tile_header(storage_manager_, uri_, file_offset); + RETURN_NOT_OK_TUPLE(st, nullopt); + auto& header = *header_opt; if (encryption_key.encryption_type() != - (EncryptionType)header.encryption_type) - return LOG_STATUS(Status_TileIOError( - "Error reading generic tile; tile is encrypted with " + - encryption_type_str((EncryptionType)header.encryption_type) + - " but given key is for " + - encryption_type_str(encryption_key.encryption_type()))); + (EncryptionType)header.encryption_type) { + return {LOG_STATUS(Status_TileIOError( + "Error reading generic tile; tile is encrypted with " + + encryption_type_str((EncryptionType)header.encryption_type) + + " but given key is for " + + encryption_type_str(encryption_key.encryption_type()))), + nullopt}; + } - RETURN_NOT_OK(configure_encryption_filter(&header, encryption_key)); + RETURN_NOT_OK_TUPLE( + configure_encryption_filter(&header, encryption_key), nullopt); const auto tile_data_offset = GenericTileHeader::BASE_SIZE + header.filter_pipeline_size; - RETURN_NOT_OK(ret->init_filtered( - header.version_number, (Datatype)header.datatype, header.cell_size, 0)); + RETURN_NOT_OK_TUPLE( + ret->init_filtered( + header.version_number, + (Datatype)header.datatype, + header.cell_size, + 0), + nullopt); // Read the tile. ret->filtered_buffer().expand(header.persisted_size); - RETURN_NOT_OK(ret->alloc_data(header.tile_size)); - RETURN_NOT_OK(storage_manager_->read( - uri_, - file_offset + tile_data_offset, - ret->filtered_buffer().data(), - header.persisted_size)); + RETURN_NOT_OK_TUPLE(ret->alloc_data(header.tile_size), nullopt); + RETURN_NOT_OK_TUPLE( + storage_manager_->read( + uri_, + file_offset + tile_data_offset, + ret->filtered_buffer().data(), + header.persisted_size), + nullopt); // Unfilter assert(ret->filtered()); - RETURN_NOT_OK(header.filters.run_reverse( - storage_manager_->stats(), - ret.get(), - nullptr, - storage_manager_->compute_tp(), - config)); + RETURN_NOT_OK_TUPLE( + header.filters.run_reverse( + storage_manager_->stats(), + ret.get(), + nullptr, + storage_manager_->compute_tp(), + config), + nullopt); assert(!ret->filtered()); - *tile = ret.release(); - return Status::Ok(); + return {Status::Ok(), std::move(ret)}; } -Status GenericTileIO::read_generic_tile_header( - const StorageManager* sm, - const URI& uri, - uint64_t file_offset, - GenericTileHeader* header) { +tuple> +GenericTileIO::read_generic_tile_header( + const StorageManager* sm, const URI& uri, uint64_t file_offset) { + GenericTileHeader header; + // Read the fixed-sized part of the header from file tdb_unique_ptr header_buff(tdb_new(Buffer)); - RETURN_NOT_OK(sm->read( - uri, file_offset, header_buff.get(), GenericTileHeader::BASE_SIZE)); + RETURN_NOT_OK_TUPLE( + sm->read( + uri, file_offset, header_buff.get(), GenericTileHeader::BASE_SIZE), + nullopt); // Read header individual values - RETURN_NOT_OK(header_buff->read(&header->version_number, sizeof(uint32_t))); - RETURN_NOT_OK(header_buff->read(&header->persisted_size, sizeof(uint64_t))); - RETURN_NOT_OK(header_buff->read(&header->tile_size, sizeof(uint64_t))); - RETURN_NOT_OK(header_buff->read(&header->datatype, sizeof(uint8_t))); - RETURN_NOT_OK(header_buff->read(&header->cell_size, sizeof(uint64_t))); - RETURN_NOT_OK(header_buff->read(&header->encryption_type, sizeof(uint8_t))); - RETURN_NOT_OK( - header_buff->read(&header->filter_pipeline_size, sizeof(uint32_t))); + RETURN_NOT_OK_TUPLE( + header_buff->read(&header.version_number, sizeof(uint32_t)), nullopt); + RETURN_NOT_OK_TUPLE( + header_buff->read(&header.persisted_size, sizeof(uint64_t)), nullopt); + RETURN_NOT_OK_TUPLE( + header_buff->read(&header.tile_size, sizeof(uint64_t)), nullopt); + RETURN_NOT_OK_TUPLE( + header_buff->read(&header.datatype, sizeof(uint8_t)), nullopt); + RETURN_NOT_OK_TUPLE( + header_buff->read(&header.cell_size, sizeof(uint64_t)), nullopt); + RETURN_NOT_OK_TUPLE( + header_buff->read(&header.encryption_type, sizeof(uint8_t)), nullopt); + RETURN_NOT_OK_TUPLE( + header_buff->read(&header.filter_pipeline_size, sizeof(uint32_t)), + nullopt); // Read header filter pipeline. header_buff->reset_size(); header_buff->reset_offset(); - RETURN_NOT_OK(sm->read( - uri, - file_offset + GenericTileHeader::BASE_SIZE, - header_buff.get(), - header->filter_pipeline_size)); + RETURN_NOT_OK_TUPLE( + sm->read( + uri, + file_offset + GenericTileHeader::BASE_SIZE, + header_buff.get(), + header.filter_pipeline_size), + nullopt); ConstBuffer cbuf(header_buff->data(), header_buff->size()); auto&& [st_filterpipeline, filterpipeline]{ - FilterPipeline::deserialize(&cbuf, header->version_number)}; - if (!st_filterpipeline.ok()) { - return st_filterpipeline; - } - header->filters = filterpipeline.value(); + FilterPipeline::deserialize(&cbuf, header.version_number)}; + RETURN_NOT_OK_TUPLE(st_filterpipeline, nullopt); + header.filters = filterpipeline.value(); - return Status::Ok(); + return {Status::Ok(), header}; } Status GenericTileIO::write_generic( diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index f6b5425fcc7..761077b2863 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -122,14 +122,12 @@ class GenericTileIO { * information about the tile, and then reads the tile data. Note that it * creates a new Tile object with the header information. * - * @param tile The tile that will hold the read data. * @param file_offset The offset in the file to read from. * @param encryption_key The encryption key to use. * @param config The storage manager's config. - * @return Status + * @return Status, Tile */ - Status read_generic( - Tile** tile, + tuple>> read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config); @@ -142,14 +140,10 @@ class GenericTileIO { * @param file_offset The offset where the header read will begin. * @param encryption_key If the array is encrypted, the private encryption * key. For unencrypted arrays, pass `nullptr`. - * @param header The header to be retrieved. - * @return Status + * @return Status, Header */ - static Status read_generic_tile_header( - const StorageManager* sm, - const URI& uri, - uint64_t file_offset, - GenericTileHeader* header); + static tuple> read_generic_tile_header( + const StorageManager* sm, const URI& uri, uint64_t file_offset); /** * Writes a tile generically to the file. This means that a header will be From acca694511a07abc5201f0e458094b65e032c92f Mon Sep 17 00:00:00 2001 From: Luc Rancourt Date: Thu, 30 Jun 2022 13:25:31 +0300 Subject: [PATCH 3/3] Addressing feedback from @ihnorton. --- tiledb/sm/fragment/fragment_metadata.cc | 94 ++++++++++---------- tiledb/sm/fragment/fragment_metadata.h | 4 +- tiledb/sm/storage_manager/storage_manager.cc | 55 ++++++------ tiledb/sm/storage_manager/storage_manager.h | 8 +- tiledb/sm/tile/generic_tile_io.cc | 23 +++-- tiledb/sm/tile/generic_tile_io.h | 2 +- 6 files changed, 96 insertions(+), 90 deletions(-) diff --git a/tiledb/sm/fragment/fragment_metadata.cc b/tiledb/sm/fragment/fragment_metadata.cc index 0ea519a2203..b98cd28b9ee 100644 --- a/tiledb/sm/fragment/fragment_metadata.cc +++ b/tiledb/sm/fragment/fragment_metadata.cc @@ -1406,9 +1406,10 @@ Status FragmentMetadata::load_fragment_min_max_sum_null_count( std::lock_guard lock(mtx_); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_min_max_sum_null_count_offset_, &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_min_max_sum_null_count_offset_); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter( "read_fragment_min_max_sum_null_count_size", buff.size()); @@ -1801,9 +1802,10 @@ Status FragmentMetadata::load_rtree(const EncryptionKey& encryption_key) { if (loaded_metadata_.rtree_) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK( - read_generic_tile_from_file(encryption_key, gt_offsets_.rtree_, &buff)); + auto&& [st, buff_opt] = + read_generic_tile_from_file(encryption_key, gt_offsets_.rtree_); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter("read_rtree_size", buff.size()); @@ -2204,9 +2206,10 @@ Status FragmentMetadata::load_tile_offsets( if (loaded_metadata_.tile_offsets_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_offsets_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_offsets_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter("read_tile_offsets_size", buff.size()); @@ -2232,9 +2235,10 @@ Status FragmentMetadata::load_tile_var_offsets( if (loaded_metadata_.tile_var_offsets_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_var_offsets_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_var_offsets_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter( "read_tile_var_offsets_size", buff.size()); @@ -2257,9 +2261,10 @@ Status FragmentMetadata::load_tile_var_sizes( if (loaded_metadata_.tile_var_sizes_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_var_sizes_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_var_sizes_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter( "read_tile_var_sizes_size", buff.size()); @@ -2282,9 +2287,10 @@ Status FragmentMetadata::load_tile_validity_offsets( if (loaded_metadata_.tile_validity_offsets_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_validity_offsets_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_validity_offsets_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter( "read_tile_validity_offsets_size", buff.size()); @@ -2307,9 +2313,10 @@ Status FragmentMetadata::load_tile_min_values( if (loaded_metadata_.tile_min_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_min_offsets_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_min_offsets_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter("read_tile_min_size", buff.size()); @@ -2331,9 +2338,10 @@ Status FragmentMetadata::load_tile_max_values( if (loaded_metadata_.tile_max_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_max_offsets_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_max_offsets_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter("read_tile_max_size", buff.size()); @@ -2355,9 +2363,10 @@ Status FragmentMetadata::load_tile_sum_values( if (loaded_metadata_.tile_sum_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_sum_offsets_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_sum_offsets_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter("read_tile_sum_size", buff.size()); @@ -2379,9 +2388,10 @@ Status FragmentMetadata::load_tile_null_count_values( if (loaded_metadata_.tile_null_count_[idx]) return Status::Ok(); - Buffer buff; - RETURN_NOT_OK(read_generic_tile_from_file( - encryption_key, gt_offsets_.tile_null_count_offsets_[idx], &buff)); + auto&& [st, buff_opt] = read_generic_tile_from_file( + encryption_key, gt_offsets_.tile_null_count_offsets_[idx]); + RETURN_NOT_OK(st); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter( "read_tile_null_count_size", buff.size()); @@ -3561,15 +3571,10 @@ Status FragmentMetadata::load_v1_v2( std::string(constants::fragment_metadata_filename)); // Read metadata GenericTileIO tile_io(storage_manager_, fragment_metadata_uri); - auto&& [st, tile_opt] = + auto&& [st, buff_opt] = tile_io.read_generic(0, encryption_key, storage_manager_->config()); RETURN_NOT_OK(st); - auto& tile = *tile_opt; - - Buffer buff; - RETURN_NOT_OK(buff.realloc(tile->size())); - buff.set_size(tile->size()); - RETURN_NOT_OK(tile->read(buff.data(), 0, buff.size())); + auto& buff = *buff_opt; storage_manager_->stats()->add_counter("read_frag_meta_size", buff.size()); @@ -3975,23 +3980,18 @@ Status FragmentMetadata::write_non_empty_domain(Buffer* buff) const { return Status::Ok(); } -Status FragmentMetadata::read_generic_tile_from_file( - const EncryptionKey& encryption_key, uint64_t offset, Buffer* buff) const { +tuple> FragmentMetadata::read_generic_tile_from_file( + const EncryptionKey& encryption_key, uint64_t offset) const { URI fragment_metadata_uri = fragment_uri_.join_path( std::string(constants::fragment_metadata_filename)); // Read metadata GenericTileIO tile_io(storage_manager_, fragment_metadata_uri); - auto&& [st, tile_opt] = + auto&& [st, buff_opt] = tile_io.read_generic(offset, encryption_key, storage_manager_->config()); - RETURN_NOT_OK(st); - auto& tile = *tile_opt; - - buff->realloc(tile->size()); - buff->set_size(tile->size()); - RETURN_NOT_OK(tile->read(buff->data(), 0, buff->size())); + RETURN_NOT_OK_TUPLE(st, nullopt); - return Status::Ok(); + return {Status::Ok(), std::move(*buff_opt)}; } Status FragmentMetadata::read_file_footer( diff --git a/tiledb/sm/fragment/fragment_metadata.h b/tiledb/sm/fragment/fragment_metadata.h index 72d0c5966e5..4741464fea7 100644 --- a/tiledb/sm/fragment/fragment_metadata.h +++ b/tiledb/sm/fragment/fragment_metadata.h @@ -1607,8 +1607,8 @@ class FragmentMetadata { * Reads the contents of a generic tile starting at the input offset, * and stores them into buffer ``buff``. */ - Status read_generic_tile_from_file( - const EncryptionKey& encryption_key, uint64_t offset, Buffer* buff) const; + tuple> read_generic_tile_from_file( + const EncryptionKey& encryption_key, uint64_t offset) const; /** * Reads the fragment metadata file footer (which contains the generic tile diff --git a/tiledb/sm/storage_manager/storage_manager.cc b/tiledb/sm/storage_manager/storage_manager.cc index 931c419925b..62300f36390 100644 --- a/tiledb/sm/storage_manager/storage_manager.cc +++ b/tiledb/sm/storage_manager/storage_manager.cc @@ -205,8 +205,10 @@ StorageManager::load_array_schemas_and_fragment_metadata( std::vector>> offsets_vectors( meta_uris.size()); auto status = parallel_for(compute_tp_, 0, meta_uris.size(), [&](size_t i) { - auto&& [st, offsets] = - load_consolidated_fragment_meta(meta_uris[i], enc_key, &f_buffs[i]); + auto&& [st, buffer_opt, offsets] = + load_consolidated_fragment_meta(meta_uris[i], enc_key); + RETURN_NOT_OK(st); + f_buffs[i] = std::move(*buffer_opt); offsets_vectors[i] = std::move(offsets.value()); return st; }); @@ -2121,7 +2123,6 @@ Status StorageManager::load_group_metadata( tuple> StorageManager::load_data_from_generic_tile( const URI& uri, const EncryptionKey& encryption_key) { - tdb_unique_ptr tile; GenericTileIO tile_io(this, uri); // Get encryption key from config @@ -2161,24 +2162,19 @@ tuple> StorageManager::load_data_from_generic_tile( nullopt); } - auto&& [st1, tile_opt] = + auto&& [st1, buff_opt] = tile_io.read_generic(0, encryption_key_cfg, config_); RETURN_NOT_OK_TUPLE(st1, nullopt); - tile = std::move(*tile_opt); + + return {Status::Ok(), std::move(*buff_opt)}; } else { - auto&& [st1, tile_opt] = tile_io.read_generic(0, encryption_key, config_); + auto&& [st1, buff_opt] = tile_io.read_generic(0, encryption_key, config_); RETURN_NOT_OK_TUPLE(st1, nullopt); - tile = std::move(*tile_opt); - } - Buffer buff; - RETURN_NOT_OK_TUPLE(buff.realloc(tile->size()), std::nullopt); - buff.set_size(tile->size()); - auto st = tile->read(buff.data(), 0, buff.size()); - - RETURN_NOT_OK_TUPLE(st, nullopt); + return {Status::Ok(), std::move(*buff_opt)}; + } - return {Status::Ok(), buff}; + assert(false); } /* ****************************** */ @@ -2264,38 +2260,41 @@ StorageManager::load_fragment_metadata( return {Status::Ok(), fragment_metadata}; } -tuple>>> +tuple< + Status, + optional, + optional>>> StorageManager::load_consolidated_fragment_meta( - const URI& uri, const EncryptionKey& enc_key, Buffer* f_buff) { + const URI& uri, const EncryptionKey& enc_key) { auto timer_se = stats_->start_timer("read_load_consolidated_frag_meta"); // No consolidated fragment metadata file if (uri.to_string().empty()) - return {Status::Ok(), nullopt}; + return {Status::Ok(), nullopt, nullopt}; - auto&& [st, buffer] = load_data_from_generic_tile(uri, enc_key); - RETURN_NOT_OK_TUPLE(st, nullopt); - buffer->swap(*f_buff); + auto&& [st, buffer_opt] = load_data_from_generic_tile(uri, enc_key); + RETURN_NOT_OK_TUPLE(st, nullopt, nullopt); + auto& buffer = *buffer_opt; - stats_->add_counter("consolidated_frag_meta_size", f_buff->size()); + stats_->add_counter("consolidated_frag_meta_size", buffer.size()); uint32_t fragment_num; - f_buff->reset_offset(); - f_buff->read(&fragment_num, sizeof(uint32_t)); + buffer.reset_offset(); + buffer.read(&fragment_num, sizeof(uint32_t)); uint64_t name_size, offset; std::string name; std::vector> ret; ret.reserve(fragment_num); for (uint32_t f = 0; f < fragment_num; ++f) { - f_buff->read(&name_size, sizeof(uint64_t)); + buffer.read(&name_size, sizeof(uint64_t)); name.resize(name_size); - f_buff->read(&name[0], name_size); - f_buff->read(&offset, sizeof(uint64_t)); + buffer.read(&name[0], name_size); + buffer.read(&offset, sizeof(uint64_t)); ret.emplace_back(name, offset); } - return {Status::Ok(), ret}; + return {Status::Ok(), buffer, ret}; } Status StorageManager::set_default_tags() { diff --git a/tiledb/sm/storage_manager/storage_manager.h b/tiledb/sm/storage_manager/storage_manager.h index 68083f5defc..6c68f062caa 100644 --- a/tiledb/sm/storage_manager/storage_manager.h +++ b/tiledb/sm/storage_manager/storage_manager.h @@ -1200,9 +1200,11 @@ class StorageManager { * @return Status, vector from the fragment name to the offset in `f_buff` * where the basic fragment metadata starts. */ - tuple>>> - load_consolidated_fragment_meta( - const URI& uri, const EncryptionKey& enc_key, Buffer* f_buff); + tuple< + Status, + optional, + optional>>> + load_consolidated_fragment_meta(const URI& uri, const EncryptionKey& enc_key); /** Block until there are zero in-progress queries. */ void wait_for_zero_in_progress(); diff --git a/tiledb/sm/tile/generic_tile_io.cc b/tiledb/sm/tile/generic_tile_io.cc index 91b050fcdf8..d8398ac4897 100644 --- a/tiledb/sm/tile/generic_tile_io.cc +++ b/tiledb/sm/tile/generic_tile_io.cc @@ -59,11 +59,11 @@ GenericTileIO::GenericTileIO(StorageManager* storage_manager, const URI& uri) /* API */ /* ****************************** */ -tuple>> GenericTileIO::read_generic( +tuple> GenericTileIO::read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config) { - tdb_unique_ptr ret(tdb_new(Tile)); + Tile tile; auto&& [st, header_opt] = read_generic_tile_header(storage_manager_, uri_, file_offset); RETURN_NOT_OK_TUPLE(st, nullopt); @@ -86,7 +86,7 @@ tuple>> GenericTileIO::read_generic( GenericTileHeader::BASE_SIZE + header.filter_pipeline_size; RETURN_NOT_OK_TUPLE( - ret->init_filtered( + tile.init_filtered( header.version_number, (Datatype)header.datatype, header.cell_size, @@ -94,27 +94,32 @@ tuple>> GenericTileIO::read_generic( nullopt); // Read the tile. - ret->filtered_buffer().expand(header.persisted_size); - RETURN_NOT_OK_TUPLE(ret->alloc_data(header.tile_size), nullopt); + tile.filtered_buffer().expand(header.persisted_size); + RETURN_NOT_OK_TUPLE(tile.alloc_data(header.tile_size), nullopt); RETURN_NOT_OK_TUPLE( storage_manager_->read( uri_, file_offset + tile_data_offset, - ret->filtered_buffer().data(), + tile.filtered_buffer().data(), header.persisted_size), nullopt); // Unfilter - assert(ret->filtered()); + assert(tile.filtered()); RETURN_NOT_OK_TUPLE( header.filters.run_reverse( storage_manager_->stats(), - ret.get(), + &tile, nullptr, storage_manager_->compute_tp(), config), nullopt); - assert(!ret->filtered()); + assert(!tile.filtered()); + + Buffer ret; + RETURN_NOT_OK_TUPLE(ret.realloc(tile.size()), std::nullopt); + ret.set_size(tile.size()); + RETURN_NOT_OK_TUPLE(tile.read(ret.data(), 0, ret.size()), nullopt); return {Status::Ok(), std::move(ret)}; } diff --git a/tiledb/sm/tile/generic_tile_io.h b/tiledb/sm/tile/generic_tile_io.h index 761077b2863..509bca9a2b4 100644 --- a/tiledb/sm/tile/generic_tile_io.h +++ b/tiledb/sm/tile/generic_tile_io.h @@ -127,7 +127,7 @@ class GenericTileIO { * @param config The storage manager's config. * @return Status, Tile */ - tuple>> read_generic( + tuple> read_generic( uint64_t file_offset, const EncryptionKey& encryption_key, const Config& config);