Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sparse unordered w/ dups reader: adding ignored tiles. #4200

Merged
merged 1 commit into from
Jul 25, 2023
Merged

Sparse unordered w/ dups reader: adding ignored tiles. #4200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 31 additions & 37 deletions test/src/unit-sparse-global-order-reader.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ struct CSparseGlobalOrderFx {
void write_delete_condition(char* value_to_delete, uint64_t value_size);
int32_t read(
bool set_subarray,
bool set_qc,
int qc_idx,
int* coords,
uint64_t* coords_size,
int* data,
Expand Down Expand Up @@ -342,7 +342,7 @@ void CSparseGlobalOrderFx::write_delete_condition(

int32_t CSparseGlobalOrderFx::read(
bool set_subarray,
bool set_qc,
int qc_idx,
int* coords,
uint64_t* coords_size,
int* data,
Expand All @@ -368,28 +368,25 @@ int32_t CSparseGlobalOrderFx::read(
CHECK(rc == TILEDB_OK);
}

if (set_qc) {
if (qc_idx != 0) {
tiledb_query_condition_t* query_condition = nullptr;
rc = tiledb_query_condition_alloc(ctx_, &query_condition);
CHECK(rc == TILEDB_OK);
int32_t val = 11;
rc = tiledb_query_condition_init(
ctx_, query_condition, "a", &val, sizeof(int32_t), TILEDB_LT);
CHECK(rc == TILEDB_OK);

// Negated query condition should produce the same results.
SECTION("- Test TILEDB_NOT") {
if (qc_idx == 1) {
int32_t val = 11;
rc = tiledb_query_condition_init(
ctx_, query_condition, "a", &val, sizeof(int32_t), TILEDB_LT);
CHECK(rc == TILEDB_OK);
} else if (qc_idx == 2) {
// Negated query condition should produce the same results.
int32_t val = 11;
tiledb_query_condition_t* qc;
rc = tiledb_query_condition_alloc(ctx_, &qc);
CHECK(rc == TILEDB_OK);
rc = tiledb_query_condition_init(
ctx_, qc, "a", &val, sizeof(int32_t), TILEDB_GE);
CHECK(rc == TILEDB_OK);

tiledb_query_condition_free(&query_condition);
query_condition = nullptr;
rc = tiledb_query_condition_alloc(ctx_, &query_condition);
CHECK(rc == TILEDB_OK);
rc = tiledb_query_condition_negate(ctx_, qc, &query_condition);
CHECK(rc == TILEDB_OK);

Expand Down Expand Up @@ -509,7 +506,7 @@ TEST_CASE_METHOD(
int data_r[5];
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);
auto rc = read(true, false, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(true, 0, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_ERR);

// Check we hit the correct error.
Expand Down Expand Up @@ -570,7 +567,7 @@ TEST_CASE_METHOD(
tiledb_query_status_t status;
rc = read(
true,
false,
0,
coords_r,
&coords_r_size,
data_r,
Expand Down Expand Up @@ -659,7 +656,7 @@ TEST_CASE_METHOD(
tiledb_query_status_t status;
rc = read(
true,
false,
0,
coords_r,
&coords_r_size,
data_r,
Expand Down Expand Up @@ -724,7 +721,7 @@ TEST_CASE_METHOD(
int data_r[5];
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);
auto rc = read(true, false, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(true, 0, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_ERR);

// Check we hit the correct error.
Expand Down Expand Up @@ -796,7 +793,7 @@ TEST_CASE_METHOD(
uint64_t data_r_size = sizeof(data_r);
rc = read(
use_subarray,
false,
0,
coords_r,
&coords_r_size,
data_r,
Expand Down Expand Up @@ -870,7 +867,7 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);
auto rc =
read(use_subarray, false, coords_r, &coords_r_size, data_r, &data_r_size);
read(use_subarray, 0, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_ERR);

// Check we hit the correct error.
Expand All @@ -896,6 +893,7 @@ TEST_CASE_METHOD(

bool use_subarray = false;
int tile_idx = 0;
int qc_idx = GENERATE(1, 2);
SECTION("- No subarray") {
use_subarray = false;
SECTION("- First tile") {
Expand Down Expand Up @@ -960,8 +958,8 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);

auto rc =
read(use_subarray, true, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(
use_subarray, qc_idx, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_OK);

// Should read two tile (6 values).
Expand All @@ -985,6 +983,7 @@ TEST_CASE_METHOD(
bool use_subarray = GENERATE(true, false);
bool dups = GENERATE(false, true);
bool extra_fragment = GENERATE(true, false);
int qc_idx = GENERATE(1, 2);

create_default_array_1d(dups);

Expand All @@ -1009,8 +1008,8 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);

auto rc =
read(use_subarray, true, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(
use_subarray, qc_idx, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_OK);

if (dups) {
Expand All @@ -1037,6 +1036,7 @@ TEST_CASE_METHOD(
reset_config();
create_default_array_1d();

int qc_idx = GENERATE(1, 2);
bool use_subarray = false;
SECTION("- No subarray") {
use_subarray = false;
Expand All @@ -1062,8 +1062,8 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);

auto rc =
read(use_subarray, true, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(
use_subarray, qc_idx, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_OK);

// One value.
Expand All @@ -1085,6 +1085,7 @@ TEST_CASE_METHOD(
create_default_array_1d(true);

bool use_subarray = false;
int qc_idx = GENERATE(1, 2);

int coords_1[] = {8, 9, 10, 11, 12, 13};
int data_1[] = {8, 9, 10, 11, 12, 13};
Expand All @@ -1105,8 +1106,8 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);

auto rc =
read(use_subarray, true, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(
use_subarray, qc_idx, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_OK);

// Should read (6 values).
Expand Down Expand Up @@ -1362,7 +1363,7 @@ TEST_CASE_METHOD(
tiledb_query_status_t status;
uint32_t rc = read(
use_subarray,
false,
0,
coords_r,
&coords_r_size,
data_r,
Expand Down Expand Up @@ -1431,14 +1432,7 @@ TEST_CASE_METHOD(
int rc;
tiledb_query_status_t status;
rc = read(
true,
false,
coords_r,
&coords_r_size,
data_r,
&data_r_size,
&query,
&array);
true, 0, coords_r, &coords_r_size, data_r, &data_r_size, &query, &array);
CHECK(rc == TILEDB_OK);

CHECK(coords_r[0] == 1);
Expand Down
59 changes: 29 additions & 30 deletions test/src/unit-sparse-unordered-with-dups-reader.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ struct CSparseUnorderedWithDupsFx {
uint64_t* a2_validity_size);
int32_t read(
bool set_subarray,
bool set_qc,
int qc_idx,
int* coords,
uint64_t* coords_size,
int* data,
Expand Down Expand Up @@ -387,7 +387,7 @@ void CSparseUnorderedWithDupsFx::write_1d_fragment_string(

int32_t CSparseUnorderedWithDupsFx::read(
bool set_subarray,
bool set_qc,
int qc_idx,
int* coords,
uint64_t* coords_size,
int* data,
Expand All @@ -413,28 +413,25 @@ int32_t CSparseUnorderedWithDupsFx::read(
CHECK(rc == TILEDB_OK);
}

if (set_qc) {
if (qc_idx != 0) {
tiledb_query_condition_t* query_condition = nullptr;
rc = tiledb_query_condition_alloc(ctx_, &query_condition);
CHECK(rc == TILEDB_OK);
int32_t val = 11;
rc = tiledb_query_condition_init(
ctx_, query_condition, "a", &val, sizeof(int32_t), TILEDB_LT);
CHECK(rc == TILEDB_OK);

// Negated query condition should produce the same results.
SECTION("- Test TILEDB_NOT") {
if (qc_idx == 1) {
int32_t val = 11;
rc = tiledb_query_condition_init(
ctx_, query_condition, "a", &val, sizeof(int32_t), TILEDB_LT);
CHECK(rc == TILEDB_OK);
} else if (qc_idx == 2) {
// Negated query condition should produce the same results.
int32_t val = 11;
tiledb_query_condition_t* qc;
rc = tiledb_query_condition_alloc(ctx_, &qc);
CHECK(rc == TILEDB_OK);
rc = tiledb_query_condition_init(
ctx_, qc, "a", &val, sizeof(int32_t), TILEDB_GE);
CHECK(rc == TILEDB_OK);

tiledb_query_condition_free(&query_condition);
query_condition = nullptr;
rc = tiledb_query_condition_alloc(ctx_, &query_condition);
CHECK(rc == TILEDB_OK);
rc = tiledb_query_condition_negate(ctx_, qc, &query_condition);
CHECK(rc == TILEDB_OK);

Expand Down Expand Up @@ -810,7 +807,7 @@ TEST_CASE_METHOD(
int data_r[5];
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);
auto rc = read(true, false, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(true, 0, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_ERR);

// Check we hit the correct error.
Expand Down Expand Up @@ -864,7 +861,7 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);
auto rc =
read(set_subarray, false, coords_r, &coords_r_size, data_r, &data_r_size);
read(set_subarray, 0, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_ERR);

// Check we hit the correct error.
Expand Down Expand Up @@ -937,7 +934,7 @@ TEST_CASE_METHOD(
uint64_t data_r_size = sizeof(data_r);
auto rc = read(
set_subarray,
false,
0,
coords_r,
&coords_r_size,
data_r,
Expand Down Expand Up @@ -1025,7 +1022,7 @@ TEST_CASE_METHOD(

auto rc = read(
use_subarray,
false,
0,
coords_r,
&coords_r_size,
data_r,
Expand Down Expand Up @@ -1100,7 +1097,7 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);
auto rc =
read(use_subarray, false, coords_r, &coords_r_size, data_r, &data_r_size);
read(use_subarray, 0, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_ERR);

// Check we hit the correct error.
Expand Down Expand Up @@ -1140,14 +1137,7 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);
auto rc = read(
false,
false,
coords_r,
&coords_r_size,
data_r,
&data_r_size,
&query,
&array);
false, 0, coords_r, &coords_r_size, data_r, &data_r_size, &query, &array);
CHECK(rc == TILEDB_OK);

// Check incomplete query status.
Expand Down Expand Up @@ -1215,6 +1205,8 @@ TEST_CASE_METHOD(

bool use_subarray = false;
int tile_idx = 0;
int qc_index = GENERATE(1, 2);
bool use_budget = GENERATE(true, false);
SECTION("- No subarray") {
use_subarray = false;
SECTION("- First tile") {
Expand All @@ -1240,6 +1232,13 @@ TEST_CASE_METHOD(
}
}

if (use_budget) {
// Two result tile (2 * ~1208) will be bigger than the budget (1500).
total_budget_ = "100000";
ratio_coords_ = "0.015";
update_config();
}

int coords_1[] = {1, 2, 3};
int data_1[] = {1, 2, 3};

Expand Down Expand Up @@ -1279,8 +1278,8 @@ TEST_CASE_METHOD(
uint64_t coords_r_size = sizeof(coords_r);
uint64_t data_r_size = sizeof(data_r);

auto rc =
read(use_subarray, true, coords_r, &coords_r_size, data_r, &data_r_size);
auto rc = read(
use_subarray, qc_index, coords_r, &coords_r_size, data_r, &data_r_size);
CHECK(rc == TILEDB_OK);

// Should read two tile (6 values).
Expand Down Expand Up @@ -1326,7 +1325,7 @@ TEST_CASE_METHOD(
uint64_t data_r_size = sizeof(data_r);
auto rc = read(
use_subarray,
false,
0,
coords_r,
&coords_r_size,
data_r,
Expand Down
5 changes: 5 additions & 0 deletions tiledb/sm/query/readers/sparse_unordered_with_dups_reader.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -388,6 +388,10 @@ bool SparseUnorderedWithDupsReader<BitmapType>::add_result_tile(
const uint64_t last_t,
const FragmentMetadata& frag_md,
ResultTilesList& result_tiles) {
if (tmp_read_state_.is_ignored_tile(f, t)) {
return false;
}

// Use either the coordinate portion of the total budget or the tile upper
// memory limit as the upper memory limit, whichever is smaller.
const uint64_t upper_memory_limit = std::min<uint64_t>(
Expand Down Expand Up @@ -545,6 +549,7 @@ void SparseUnorderedWithDupsReader<BitmapType>::clean_tile_list(
while (it != result_tiles.end()) {
auto f = it->frag_idx();
if (it->result_num() == 0) {
tmp_read_state_.add_ignored_tile(*it);
remove_result_tile(f, result_tiles, it++);
} else {
it++;
Expand Down
Loading