Skip to content
Permalink
Browse files
fix issue #236
  • Loading branch information
AlexanderSaydakov committed Sep 17, 2021
1 parent 18455ec commit 8ded0a8949a7602d795e9f807c931bc3b29f86ca
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 3 deletions.
@@ -350,7 +350,7 @@ frequent_items_sketch<T, W, H, E, S, A> frequent_items_sketch<T, W, H, E, S, A>:
check_serial_version(serial_version);
check_family_id(family_id);
check_size(lg_cur_size, lg_max_size);
ensure_minimum_memory(size, 1ULL << preamble_longs);
ensure_minimum_memory(size, preamble_longs * sizeof(uint64_t));

frequent_items_sketch<T, W, H, E, S, A> sketch(lg_max_size, lg_cur_size, allocator);
if (!is_empty) {
@@ -575,7 +575,7 @@ kll_sketch<T, C, S, A> kll_sketch<T, C, S, A>::deserialize(const void* bytes, si
check_preamble_ints(preamble_ints, flags_byte);
check_serial_version(serial_version);
check_family_id(family_id);
ensure_minimum_memory(size, 1ULL << preamble_ints);
ensure_minimum_memory(size, preamble_ints * sizeof(uint32_t));

const bool is_empty(flags_byte & (1 << flags::IS_EMPTY));
if (is_empty) return kll_sketch<T, C, S, A>(k, allocator);
@@ -279,6 +279,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
REQUIRE(s.tellg() == s.tellp());
REQUIRE(sketch2.is_empty() == sketch.is_empty());
REQUIRE(sketch2.is_estimation_mode() == sketch.is_estimation_mode());
REQUIRE(sketch2.get_n() == sketch.get_n());
@@ -304,7 +305,7 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch2.get_normalized_rank_error(true) == sketch.get_normalized_rank_error(true));
}

SECTION("serialize deserialize one item") {
SECTION("stream serialize deserialize one item") {
kll_float_sketch sketch(200, 0);
sketch.update(1.0f);
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
@@ -324,6 +325,24 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch2.get_rank(2) == 1.0);
}

SECTION("bytes serialize deserialize one item") {
kll_float_sketch sketch(200, 0);
sketch.update(1.0f);
auto bytes = sketch.serialize();
REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
REQUIRE_FALSE(sketch2.is_empty());
REQUIRE_FALSE(sketch2.is_estimation_mode());
REQUIRE(sketch2.get_n() == 1);
REQUIRE(sketch2.get_num_retained() == 1);
REQUIRE(sketch2.get_min_value() == 1.0);
REQUIRE(sketch2.get_max_value() == 1.0);
REQUIRE(sketch2.get_quantile(0.5) == 1.0);
REQUIRE(sketch2.get_rank(1) == 0.0);
REQUIRE(sketch2.get_rank(2) == 1.0);
}

SECTION("deserialize one item v1") {
std::ifstream is;
is.exceptions(std::ios::failbit | std::ios::badbit);
@@ -337,6 +356,42 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(sketch.get_max_value() == 1.0);
}

SECTION("stream serialize deserialize three items") {
kll_float_sketch sketch(200, 0);
sketch.update(1.0f);
sketch.update(2.0f);
sketch.update(3.0f);
std::stringstream s(std::ios::in | std::ios::out | std::ios::binary);
sketch.serialize(s);
REQUIRE(static_cast<size_t>(s.tellp()) == sketch.get_serialized_size_bytes());
auto sketch2 = kll_float_sketch::deserialize(s, test_allocator<float>(0));
REQUIRE(static_cast<size_t>(s.tellp()) == sketch2.get_serialized_size_bytes());
REQUIRE(s.tellg() == s.tellp());
REQUIRE_FALSE(sketch2.is_empty());
REQUIRE_FALSE(sketch2.is_estimation_mode());
REQUIRE(sketch2.get_n() == 3);
REQUIRE(sketch2.get_num_retained() == 3);
REQUIRE(sketch2.get_min_value() == 1.0);
REQUIRE(sketch2.get_max_value() == 3.0);
}

SECTION("bytes serialize deserialize one item") {
kll_float_sketch sketch(200, 0);
sketch.update(1.0f);
sketch.update(2.0f);
sketch.update(3.0f);
auto bytes = sketch.serialize();
REQUIRE(bytes.size() == sketch.get_serialized_size_bytes());
auto sketch2 = kll_float_sketch::deserialize(bytes.data(), bytes.size(), 0);
REQUIRE(bytes.size() == sketch2.get_serialized_size_bytes());
REQUIRE_FALSE(sketch2.is_empty());
REQUIRE_FALSE(sketch2.is_estimation_mode());
REQUIRE(sketch2.get_n() == 3);
REQUIRE(sketch2.get_num_retained() == 3);
REQUIRE(sketch2.get_min_value() == 1.0);
REQUIRE(sketch2.get_max_value() == 3.0);
}

SECTION("stream serialize deserialize many floats") {
kll_float_sketch sketch(200, 0);
const int n = 1000;
@@ -702,6 +757,15 @@ TEST_CASE("kll sketch", "[kll_sketch]") {
REQUIRE(kll_sketch<std::string>::get_max_serialized_size_bytes(200, 1000000000, 4) == 3160);
}

SECTION("issue #236") {
kll_sketch<int8_t> kll;
kll.update(1);
kll.update(2);
kll.update(3);
auto blob = kll.serialize();
auto kll2 = kll_sketch<int8_t>::deserialize(blob.data(), blob.size());
}

// cleanup
if (test_allocator_total_bytes != 0) {
REQUIRE(test_allocator_total_bytes == 0);

0 comments on commit 8ded0a8

Please sign in to comment.