Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Integrate std::pmr memory tracking for class ArraySchema. #4696

Merged
merged 16 commits into from
Feb 23, 2024
2 changes: 1 addition & 1 deletion test/src/unit-ReadCellSlabIter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ TEST_CASE_METHOD(
shared_ptr<FragmentMetadata> fragment = make_shared<FragmentMetadata>(
HERE(),
nullptr,
create_test_memory_tracker(),
tiledb::test::create_test_memory_tracker(),
array_->array_->array_schema_latest_ptr(),
URI(),
std::make_pair<uint64_t, uint64_t>(0, 0),
Expand Down
5 changes: 4 additions & 1 deletion test/src/unit-Reader.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,15 @@
*/

#include "test/support/src/helpers.h"
#include "test/support/src/mem_helpers.h"
#include "test/support/src/vfs_helpers.h"
#include "tiledb/common/common.h"
#include "tiledb/common/dynamic_memory/dynamic_memory.h"
#include "tiledb/common/heap_memory.h"
#include "tiledb/common/logger.h"
#include "tiledb/common/memory_tracker.h"
#include "tiledb/sm/c_api/tiledb_struct_def.h"
#include "tiledb/sm/enums/array_type.h"
#include "tiledb/sm/enums/encryption_type.h"
#include "tiledb/sm/misc/types.h"
#include "tiledb/sm/query/legacy/reader.h"
Expand Down Expand Up @@ -250,7 +252,8 @@ TEST_CASE_METHOD(
CHECK(dom->add_dimension(d1).ok());
CHECK(dom->add_dimension(d2).ok());

auto schema = make_shared<ArraySchema>(HERE());
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);
CHECK(schema->set_domain(dom).ok());

std::vector<shared_ptr<FragmentMetadata>> fragments;
Expand Down
2 changes: 2 additions & 0 deletions test/src/unit-capi-array.cc
Original file line number Diff line number Diff line change
Expand Up @@ -2586,8 +2586,10 @@ TEST_CASE_METHOD(
1,
&buff);
REQUIRE(rc == TILEDB_OK);
auto memory_tracker = ctx_->resources().create_memory_tracker();
auto st = tiledb::sm::serialization::array_deserialize(
array->array_.get(),
memory_tracker,
tiledb::sm::SerializationType::CAPNP,
buff->buffer(),
ctx_->storage_manager());
Expand Down
5 changes: 4 additions & 1 deletion test/src/unit-cppapi-schema-evolution.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
*/

#include <test/support/tdb_catch.h>
#include "test/support/src/mem_helpers.h"
#include "tiledb/sm/array_schema/array_schema.h"
#include "tiledb/sm/array_schema/array_schema_evolution.h"
#include "tiledb/sm/array_schema/attribute.h"
Expand Down Expand Up @@ -808,7 +809,9 @@ TEST_CASE(
ase->set_timestamp_range(std::make_pair(1, 1));

auto schema = make_shared<tiledb::sm::ArraySchema>(
HERE(), tiledb::sm::ArrayType::SPARSE);
HERE(),
tiledb::test::create_test_memory_tracker(),
tiledb::sm::ArrayType::SPARSE);
auto dim = make_shared<tiledb::sm::Dimension>(
HERE(), "dim1", tiledb::sm::Datatype::INT32);
int range[2] = {0, 1000};
Expand Down
82 changes: 51 additions & 31 deletions test/src/unit-enumerations.cc
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

#include <sstream>

#include "test/support/src/mem_helpers.h"
#include "test/support/tdb_catch.h"
#include "tiledb/common/memory_tracker.h"
#include "tiledb/sm/array/array.h"
Expand Down Expand Up @@ -115,7 +116,7 @@ struct EnumerationFx {
shared_ptr<ArraySchema> get_array_schema_latest();

// Serialization helpers
ArraySchema ser_des_array_schema(
shared_ptr<ArraySchema> ser_des_array_schema(
shared_ptr<const ArraySchema> schema,
bool client_side,
SerializationType stype);
Expand Down Expand Up @@ -1189,15 +1190,17 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Add Enumeration - Enumeration nullptr Error",
"[enumeration][array-schema][error]") {
auto schema = make_shared<ArraySchema>(HERE());
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);
abigalekim marked this conversation as resolved.
Show resolved Hide resolved
REQUIRE_THROWS(schema->add_enumeration(nullptr));
}

TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Add Basic Enumeration",
"[enumeration][array-schema][basic]") {
auto schema = make_shared<ArraySchema>(HERE());
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);

std::vector<int> values = {1, 2, 3, 4, 5};
auto enmr = create_enumeration(values);
Expand All @@ -1212,7 +1215,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Get Enumeration",
"[enumeration][array-schema][get]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);

std::vector<int> values = {1, 2, 3, 4, 5};
auto enmr1 = create_enumeration(values);
Expand All @@ -1226,15 +1230,17 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Get Missing Enumeration Error",
"[enumeration][array-schema][error]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::SPARSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::SPARSE);
REQUIRE_THROWS(schema->get_enumeration("not_an_enumeration"));
}

TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Add Enumeration with Existing Enumeration of same Name",
"[enumeration][array-schema][eror]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::SPARSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::SPARSE);
std::vector<int> values = {1, 2, 3, 4, 5};
auto enmr = create_enumeration(values);

Expand All @@ -1246,7 +1252,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Add Attribute with Missing Enumeration Error",
"[enumeration][array-schema][eror]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::SPARSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::SPARSE);
auto attr = make_shared<Attribute>(HERE(), "an_attr", Datatype::INT32);
attr->set_enumeration_name("not_an_enumeration");
REQUIRE(!schema->add_attribute(attr).ok());
Expand All @@ -1256,7 +1263,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Get All Enumeration Names Empty",
"[enumeration][array-schema][get-all][empty]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);
auto enmr_names = schema->get_enumeration_names();
REQUIRE(enmr_names.size() == 0);
}
Expand All @@ -1265,7 +1273,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Get All Enumeration Names",
"[enumeration][array-schema][get-all]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);

std::vector<float> values = {1.0f, 1.1f, 1.2f, 1.3f, 1.4f};
auto enmr1 = create_enumeration(values);
Expand All @@ -1284,7 +1293,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Attribute with Invalid Datatype",
"[enumeration][array-schema][error][bad-attr-datatype]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);

std::vector<int> values = {1, 2, 3, 4, 5};
auto enmr = create_enumeration(values);
Expand All @@ -1299,7 +1309,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Attribute with Invalid Cell Val Num",
"[enumeration][array-schema][error][bad-attr-cell-val-num]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);

std::vector<int> values = {1, 2, 3, 4, 5};
auto enmr = create_enumeration(values);
Expand All @@ -1315,15 +1326,17 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Store nullptr Enumeration Error",
"[enumeration][array-schema][error][store-nullptr-enumeration]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);
REQUIRE_THROWS(schema->store_enumeration(nullptr));
}

TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Store Enumeration Error",
"[enumeration][array-schema][error][store-unknown-enumeration]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);
std::vector<int> values = {1, 2, 3, 4, 5};
auto enmr =
create_enumeration(values, false, Datatype::INT32, "unknown_enmr");
Expand All @@ -1334,7 +1347,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Store Enumeration Error - Already Loaded",
"[enumeration][array-schema][error][store-loaded-enumeration]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::DENSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::DENSE);

std::vector<uint32_t> values = {0, 1, 2, 100000000};
auto enmr = create_enumeration(values);
Expand All @@ -1350,7 +1364,8 @@ TEST_CASE_METHOD(
EnumerationFx,
"ArraySchema - Attribute Get Enumeration Name From Attribute",
"[enumeration][array-schema][has-enumeration]") {
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::SPARSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::SPARSE);

std::vector<std::string> values = {"a", "spot", "of", "tea", "perhaps?"};
auto enmr = create_enumeration(values);
Expand Down Expand Up @@ -1427,10 +1442,8 @@ TEST_CASE_METHOD(
"[enumeration][array-schema][copy-ctor]") {
auto schema = create_schema();

// Check that the schema is valid and that we can copy it using the
// copy constructor.
// Check that the schema is valid.
CHECK_NOTHROW(schema->check(cfg_));
CHECK_NOTHROW(make_shared<ArraySchema>(HERE(), *(schema.get())));
}

TEST_CASE_METHOD(
Expand Down Expand Up @@ -1509,7 +1522,8 @@ TEST_CASE_METHOD(
auto array = get_array(QueryType::READ);
array->load_all_enumerations();

auto schema = make_shared<ArraySchema>(HERE(), array->array_schema_latest());
auto schema = array->array_schema_latest().clone();

auto enmr = create_empty_enumeration(Datatype::INT32, 1, false, "test_enmr");

auto matcher = Catch::Matchers::ContainsSubstring(
Expand All @@ -1526,7 +1540,7 @@ TEST_CASE_METHOD(
auto array = get_array(QueryType::READ);
array->load_all_enumerations();

auto schema = make_shared<ArraySchema>(HERE(), array->array_schema_latest());
auto schema = array->array_schema_latest().clone();
auto enmr1 = schema->get_enumeration("test_enmr");

std::vector<std::string> extra_values = {"manatee", "narwhal", "oppossum"};
Expand Down Expand Up @@ -2082,11 +2096,11 @@ TEST_CASE_METHOD(
auto schema2 = ser_des_array_schema(schema1, client_side, ser_type);

auto all_names1 = schema1->get_enumeration_names();
auto all_names2 = schema2.get_enumeration_names();
auto all_names2 = schema2->get_enumeration_names();
REQUIRE(vec_cmp(all_names1, all_names2));

auto loaded_names1 = schema1->get_loaded_enumeration_names();
auto loaded_names2 = schema2.get_loaded_enumeration_names();
auto loaded_names2 = schema2->get_loaded_enumeration_names();
REQUIRE(vec_cmp(loaded_names1, loaded_names2));

// This is a new schema in RAM, so the loaded names should be the same
Expand All @@ -2108,13 +2122,13 @@ TEST_CASE_METHOD(
auto schema2 = ser_des_array_schema(schema1, client_side, ser_type);

auto all_names1 = schema1->get_enumeration_names();
auto all_names2 = schema2.get_enumeration_names();
auto all_names2 = schema2->get_enumeration_names();
REQUIRE(vec_cmp(all_names1, all_names2));

// This schema was deserialized from disk without any enumerations loaded
// so both of these should be empty.
auto loaded_names1 = schema1->get_loaded_enumeration_names();
auto loaded_names2 = schema2.get_loaded_enumeration_names();
auto loaded_names2 = schema2->get_loaded_enumeration_names();

REQUIRE(loaded_names1.empty());
REQUIRE(loaded_names2.empty());
Expand Down Expand Up @@ -2149,7 +2163,7 @@ TEST_CASE_METHOD(
auto schema2 = ser_des_array_schema(schema1, client_side, ser_type);

auto all_names1 = schema1->get_enumeration_names();
auto all_names2 = schema2.get_enumeration_names();
auto all_names2 = schema2->get_enumeration_names();
REQUIRE(vec_cmp(all_names1, all_names2));
}

Expand Down Expand Up @@ -2672,7 +2686,8 @@ std::vector<T> EnumerationFx::as_vector(shared_ptr<const Enumeration> enmr) {

shared_ptr<ArraySchema> EnumerationFx::create_schema() {
// Create a schema to serialize
auto schema = make_shared<ArraySchema>(HERE(), ArrayType::SPARSE);
auto schema = make_shared<ArraySchema>(
HERE(), tiledb::test::create_test_memory_tracker(), ArrayType::SPARSE);

auto dim = make_shared<Dimension>(HERE(), "dim1", Datatype::INT32);
int range[2] = {0, 1000};
Expand Down Expand Up @@ -2729,14 +2744,15 @@ shared_ptr<ArraySchema> EnumerationFx::get_array_schema_latest() {

#ifdef TILEDB_SERIALIZATION

ArraySchema EnumerationFx::ser_des_array_schema(
shared_ptr<ArraySchema> EnumerationFx::ser_des_array_schema(
shared_ptr<const ArraySchema> schema,
bool client_side,
SerializationType stype) {
Buffer buf;
throw_if_not_ok(serialization::array_schema_serialize(
*(schema.get()), stype, &buf, client_side));
return serialization::array_schema_deserialize(stype, buf);
return serialization::array_schema_deserialize(
stype, tiledb::test::create_test_memory_tracker(), buf);
}

shared_ptr<ArraySchemaEvolution> EnumerationFx::ser_des_array_schema_evolution(
Expand Down Expand Up @@ -2779,13 +2795,17 @@ void EnumerationFx::ser_des_array(
SerializationType stype) {
Buffer buf;
throw_if_not_ok(serialization::array_serialize(in, stype, &buf, client_side));
throw_if_not_ok(
serialization::array_deserialize(out, stype, buf, ctx.storage_manager()));
throw_if_not_ok(serialization::array_deserialize(
out,
tiledb::test::create_test_memory_tracker(),
stype,
buf,
ctx.storage_manager()));
}

#else // No TILEDB_SERIALIZATION

ArraySchema EnumerationFx::ser_des_array_schema(
shared_ptr<ArraySchema> EnumerationFx::ser_des_array_schema(
shared_ptr<const ArraySchema>, bool, SerializationType) {
throw std::logic_error("Serialization not enabled.");
}
Expand Down