Skip to content

[Test](bitmap) Harden BitmapValue serialization test #65491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
linrrzqqq wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[Test](bitmap) Harden BitmapValue serialization test #65491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 114 additions & 10 deletions be/test/core/value/bitmap_value_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,15 @@
#include <gtest/gtest-test-part.h>
#include <roaring/roaring.h>

#include <algorithm>
#include <cstdint>
#include <string>
#include <vector>

#include "gtest/gtest.h"
#include "gtest/gtest_pred_impl.h"
#include "util/coding.h"
#include "util/url_coding.h"

namespace doris {
using roaring::Roaring;
Expand Down Expand Up @@ -503,6 +506,20 @@ void check_bitmap_equal(const BitmapValue& left, const BitmapValue& right) {
}
}

std::vector<uint64_t> sorted_bitmap_values(const BitmapValue& bitmap) {
std::vector<uint64_t> values;
for (auto v : bitmap) {
values.emplace_back(v);
}
std::sort(values.begin(), values.end());
return values;
}

void check_bitmap_values(const BitmapValue& bitmap, std::vector<uint64_t> expected) {
std::sort(expected.begin(), expected.end());
EXPECT_EQ(sorted_bitmap_values(bitmap), expected);
}

TEST(BitmapValueTest, write_read) {
config::enable_set_in_bitmap_value = true;
BitmapValue bitmap_empty;
Expand Down Expand Up @@ -966,14 +983,31 @@ std::string convert_bitmap_to_string(BitmapValue& bitmap) {
return buf;
}

std::string decode_base64_to_string(const std::string& base64) {
std::string decoded;
EXPECT_TRUE(base64_decode(base64, &decoded));
return decoded;
}

BitmapValue deserialize_bitmap_from_string(const std::string& buffer) {
BitmapValue bitmap;
EXPECT_TRUE(bitmap.deserialize(buffer.data()));
return bitmap;
}

TEST(BitmapValueTest, bitmap_serde) {
auto old_enable_set = config::enable_set_in_bitmap_value;
auto old_serialize_version = config::bitmap_serialize_version;
config::enable_set_in_bitmap_value = false;
config::bitmap_serialize_version = 1;

{ // EMPTY
BitmapValue empty;
std::string buffer = convert_bitmap_to_string(empty);
std::string expect_buffer(1, BitmapTypeCode::EMPTY);
EXPECT_EQ(expect_buffer, buffer);

BitmapValue out(buffer.data());
BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(0, out.cardinality());
}
{ // SINGLE32
Expand All @@ -984,18 +1018,19 @@ TEST(BitmapValueTest, bitmap_serde) {
put_fixed32_le(&expect_buffer, i);
EXPECT_EQ(expect_buffer, buffer);

BitmapValue out(buffer.data());
BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(1, out.cardinality());
EXPECT_TRUE(out.contains(i));
}
{ // BITMAP32
BitmapValue bitmap32({0, UINT32_MAX});
BitmapValue bitmap32({1, 9999999});
std::string buffer = convert_bitmap_to_string(bitmap32);
EXPECT_EQ(decode_base64_to_string("AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y="), buffer);

BitmapValue out(buffer.data());
BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(2, out.cardinality());
EXPECT_TRUE(out.contains(0));
EXPECT_TRUE(out.contains(UINT32_MAX));
EXPECT_TRUE(out.contains(1));
EXPECT_TRUE(out.contains(9999999));
}
{ // SINGLE64
uint64_t i = static_cast<uint64_t>(UINT32_MAX) + 1;
Expand All @@ -1005,19 +1040,88 @@ TEST(BitmapValueTest, bitmap_serde) {
put_fixed64_le(&expect_buffer, i);
EXPECT_EQ(expect_buffer, buffer);

BitmapValue out(buffer.data());
BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(1, out.cardinality());
EXPECT_TRUE(out.contains(i));
}
{ // BITMAP64
BitmapValue bitmap64({0, static_cast<uint64_t>(UINT32_MAX) + 1});
BitmapValue bitmap64({1, static_cast<uint64_t>(UINT32_MAX) + 1});
std::string buffer = convert_bitmap_to_string(bitmap64);
EXPECT_EQ(decode_base64_to_string(
"BAIAAAAAOjAAAAEAAAAAAAAAEAAAAAEAAQAAADowAAABAAAAAAAAABAAAAAAAA=="),
buffer);

BitmapValue out(buffer.data());
BitmapValue out = deserialize_bitmap_from_string(buffer);
EXPECT_EQ(2, out.cardinality());
EXPECT_TRUE(out.contains(0));
EXPECT_TRUE(out.contains(1));
EXPECT_TRUE(out.contains(static_cast<uint64_t>(UINT32_MAX) + 1));
}
{ // BITMAP32_V2
config::bitmap_serialize_version = 2;
std::vector<uint64_t> bits32 {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32};
BitmapValue bitmap32(bits32);
std::string buffer = convert_bitmap_to_string(bitmap32);
EXPECT_EQ(decode_base64_to_string("DAI7MAAAAQAAIAABAAAAIAA="), buffer);

BitmapValue out = deserialize_bitmap_from_string(buffer);
check_bitmap_values(out, bits32);
}
{ // BITMAP64_V2
std::vector<uint64_t> bits64 {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10,
11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21,
22, 23, 24, 25, 26, 27, 28, 29, 30, 31, static_cast<uint64_t>(UINT32_MAX) + 1};
BitmapValue bitmap64(bits64);
std::string buffer = convert_bitmap_to_string(bitmap64);
EXPECT_EQ(decode_base64_to_string("DQIAAAAAAjswAAABAAAfAAEAAAAfAAEAAAABAQAAAAAAAAA="),
buffer);

BitmapValue out = deserialize_bitmap_from_string(buffer);
check_bitmap_values(out, bits64);
}
{ // SET serialization is semantic-only because hash-set iteration order is not stable.
config::enable_set_in_bitmap_value = true;
config::bitmap_serialize_version = 1;
std::vector<uint64_t> values {UINT64_MAX, 0, static_cast<uint64_t>(UINT32_MAX) + 1, 7, 1};
BitmapValue set_bitmap;
for (auto value : values) {
set_bitmap.add(value);
}
EXPECT_EQ(set_bitmap.get_type_code(), BitmapTypeCode::SET);

std::string buffer = convert_bitmap_to_string(set_bitmap);
BitmapValue out = deserialize_bitmap_from_string(buffer);
check_bitmap_values(out, values);
}
{ // Deserializing historical SET bytes must not depend on their serialized payload order.
config::enable_set_in_bitmap_value = true;
std::vector<uint64_t> values {1, 9999999};
std::string set_v1_insert_order = decode_base64_to_string("BQIBAAAAAAAAAH+WmAAAAAAA");
std::string set_v1_reverse_order = decode_base64_to_string("BQJ/lpgAAAAAAAEAAAAAAAAA");
std::string set_v2_reverse_order = decode_base64_to_string("CgIAAAB/lpgAAAAAAAEAAAAAAAAA");

BitmapValue set_in_insert_order = deserialize_bitmap_from_string(set_v1_insert_order);
BitmapValue set_in_reverse_order = deserialize_bitmap_from_string(set_v1_reverse_order);
BitmapValue set_v2_in_reverse_order = deserialize_bitmap_from_string(set_v2_reverse_order);

check_bitmap_values(set_in_insert_order, values);
check_bitmap_values(set_in_reverse_order, values);
check_bitmap_values(set_v2_in_reverse_order, values);
}
{ // Historical SET bytes with mixed 32-bit and 64-bit values must remain readable.
config::enable_set_in_bitmap_value = true;
std::vector<uint64_t> values {UINT64_MAX, 0, static_cast<uint64_t>(UINT32_MAX) + 1, 7, 1};
std::string set_v1_mixed =
decode_base64_to_string("BQX//////////wAAAAAAAAAAAAAAAAEAAAAHAAAAAAAAAAEAAAAAAAAA");

BitmapValue out = deserialize_bitmap_from_string(set_v1_mixed);
check_bitmap_values(out, values);
}

config::enable_set_in_bitmap_value = old_enable_set;
config::bitmap_serialize_version = old_serialize_version;
}

// Forked from CRoaring's UT of Roaring64Map
Expand Down
31 changes: 27 additions & 4 deletions be/test/exprs/function/function_bitmap_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include "core/types.h"
#include "core/value/bitmap_value.h"
#include "exprs/function/function_test_util.h"
#include "util/url_coding.h"

namespace doris {

Expand Down Expand Up @@ -98,6 +99,14 @@ namespace doris::config {
DECLARE_Bool(enable_set_in_bitmap_value);
}

std::string encode_bitmap_to_base64(const BitmapValue& bitmap) {
std::string serialized(bitmap.getSizeInBytes(), '\0');
bitmap.write_to(serialized.data());
std::string encoded;
base64_encode(serialized, &encoded);
return encoded;
}

TEST(function_bitmap_test, function_bitmap_to_base64) {
config::Register::Field field("bool", "enable_set_in_bitmap_value",
&config::enable_set_in_bitmap_value, "false", false);
Expand Down Expand Up @@ -175,10 +184,8 @@ TEST(function_bitmap_test, function_bitmap_to_base64) {
{
DataSet data_set = {
{{&bitmap32_1}, std::string("AQEAAAA=")},
{{&bitmap32_2}, std::string("BQIBAAAAAAAAAH+WmAAAAAAA")},
{{&bitmap32_3}, std::string("AjswAAABAAAgAAEAAAAgAA==")},
{{&bitmap64_1}, std::string("AwAAAAABAAAA")},
{{&bitmap64_2}, std::string("BQIAAAAAAQAAAAEAAAAAAAAA")},
{{&bitmap64_3},
std::string("BAIAAAAAOzAAAAEAAB8AAQAAAB8AAQAAADowAAABAAAAAAAAABAAAAAAAA==")},
{{&empty_bitmap}, std::string("AA==")},
Expand All @@ -188,13 +195,20 @@ TEST(function_bitmap_test, function_bitmap_to_base64) {
}

{
std::string base64("BQQAAAAAAAAAAAEAAAAAAAAAAgAAAAAAAAADAAAAAAAAAA==");
// SET serialization depends on hash-set iteration order, so do not pin a hard-coded
// base64 string here. The contract is that the current serialization can round-trip.
DataSet data_set = {{{&bitmap32_2}, encode_bitmap_to_base64(bitmap32_2)},
{{&bitmap64_2}, encode_bitmap_to_base64(bitmap64_2)}};
static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
}

{
BitmapValue bitmap;
bitmap.add(0);
bitmap.add(1);
bitmap.add(2);
bitmap.add(3);
DataSet data_set = {{{&bitmap}, base64}};
DataSet data_set = {{{&bitmap}, encode_bitmap_to_base64(bitmap)}};
static_cast<void>(check_function<DataTypeString, true>(func_name, input_types, data_set));
}

Expand Down Expand Up @@ -291,6 +305,15 @@ TEST(function_bitmap_test, function_bitmap_from_base64) {
DataSet data_set = {{{base64}, &bitmap}};
static_cast<void>(check_function<DataTypeBitMap, true>(func_name, input_types, data_set));
}
{
// Historical SET bytes may store the same elements in different orders; decoding must
// preserve the bitmap contents instead of relying on the serialized payload order.
std::string set_v1_reversed("BQJ/lpgAAAAAAAEAAAAAAAAA");
std::string set_v2_reversed("CgIAAAB/lpgAAAAAAAEAAAAAAAAA");
BitmapValue bitmap({1, 9999999});
DataSet data_set = {{{set_v1_reversed}, &bitmap}, {{set_v2_reversed}, &bitmap}};
static_cast<void>(check_function<DataTypeBitMap, true>(func_name, input_types, data_set));
}
{
EXPECT_TRUE(config::set_config("bitmap_serialize_version", "1", false, true).ok());

Expand Down
12 changes: 12 additions & 0 deletions regression-test/data/query_p0/sql_functions/bitmap_functions/test_bitmap_function.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -537,6 +537,18 @@ true
-- !sql_bitmap_base64_nereids8 --
1

-- !sql_bitmap_base64_history_set_v1 --
1,9999999

-- !sql_bitmap_base64_history_set_v1_reversed --
1,9999999

-- !sql_bitmap_base64_history_set_v2_reversed --
1,9999999

-- !sql_bitmap_base64_history_set_v1_mixed --
0,1,7,4294967296,18446744073709551615

-- !sql_bitmap_base64_nereids9 --

0
Expand Down
11 changes: 4 additions & 7 deletions regression-test/suites/doc/sql-manual/sql-functions/doc_bitmap_functions_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,21 +31,18 @@ suite("doc_bitmap_functions_test") {
''')

def bitmapToBase64Single = sql '''
SELECT bitmap_to_base64(to_bitmap(1));
SELECT bitmap_to_string(bitmap_from_base64(bitmap_to_base64(to_bitmap(1))));
'''
def bitmapToBase64SingleExpected = ["BQEBAAAAAAAAAA==", "AQEAAAA="]
assertTrue(bitmapToBase64SingleExpected.contains(bitmapToBase64Single[0][0].toString()),
assertEquals("1", bitmapToBase64Single[0][0].toString(),
"Unexpected bitmap_to_base64 single result: ${bitmapToBase64Single}")
testFoldConst('''
SELECT bitmap_to_base64(to_bitmap(1));
''')

def bitmapToBase64Multi = sql '''
SELECT bitmap_to_base64(bitmap_from_string("1,9999999"));
SELECT bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("1,9999999"))));
'''
def bitmapToBase64MultiExpected = ["BQIBAAAAAAAAAH+WmAAAAAAA",
"AjowAAACAAAAAAAAAJgAAAAYAAAAGgAAAAEAf5Y="]
assertTrue(bitmapToBase64MultiExpected.contains(bitmapToBase64Multi[0][0].toString()),
assertEquals("1,9999999", bitmapToBase64Multi[0][0].toString(),
"Unexpected bitmap_to_base64 multi result: ${bitmapToBase64Multi}")
testFoldConst('''
SELECT bitmap_to_base64(bitmap_from_string("1,9999999"));
Expand Down
6 changes: 6 additions & 0 deletions regression-test/suites/query_p0/sql_functions/bitmap_functions/test_bitmap_function.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -777,6 +777,12 @@ suite("test_bitmap_function") {
qt_sql_bitmap_base64_nereids6 """ select bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32")))) """
qt_sql_bitmap_base64_nereids7 """ select bitmap_to_string(bitmap_from_base64(bitmap_to_base64(bitmap_from_string("0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,4294967296")))) """
qt_sql_bitmap_base64_nereids8 """ select bitmap_to_string(bitmap_from_base64(bitmap_to_base64(to_bitmap(1)))); """
// SET serialization order is not stable because it depends on hash-set iteration order.
// These fixed historical bytes verify from_base64 semantics rather than bitmap_to_base64 bytes.
qt_sql_bitmap_base64_history_set_v1 """ select bitmap_to_string(bitmap_from_base64("BQIBAAAAAAAAAH+WmAAAAAAA")); """
qt_sql_bitmap_base64_history_set_v1_reversed """ select bitmap_to_string(bitmap_from_base64("BQJ/lpgAAAAAAAEAAAAAAAAA")); """
qt_sql_bitmap_base64_history_set_v2_reversed """ select bitmap_to_string(bitmap_from_base64("CgIAAAB/lpgAAAAAAAEAAAAAAAAA")); """
qt_sql_bitmap_base64_history_set_v1_mixed """ select bitmap_to_string(bitmap_from_base64("BQX//////////wAAAAAAAAAAAAAAAAEAAAAHAAAAAAAAAAEAAAAAAAAA")); """

// test nullable
sql """ DROP TABLE IF EXISTS test_bitmap_base64 """
Expand Down
Loading