Skip to content
Permalink
Browse files

fix an issue in memory map manager introducing a new chunk despite si… (

#115)

fix an issue in memory map manager introducing a new chunk despite size 0
  • Loading branch information...
hendrikmuhs committed Apr 16, 2019
1 parent f27619a commit 10b6717a4d0a26df0c01ae993308aff1aa7f0ccf
@@ -93,15 +93,17 @@ class MemoryMapManager final {
size_t chunk_number = offset / chunk_size_;
size_t chunk_offset = offset % chunk_size_;

void* chunk_address = GetChunk(chunk_number);
void* chunk_address_part2 = GetChunk(chunk_number + 1);

size_t first_chunk_size = std::min(buffer_length, chunk_size_ - chunk_offset);
size_t second_chunk_size = buffer_length - first_chunk_size;

void* chunk_address = GetChunk(chunk_number);
std::memcpy(buffer, reinterpret_cast<char*>(chunk_address) + chunk_offset, first_chunk_size);
std::memcpy(reinterpret_cast<char*>(buffer) + first_chunk_size, reinterpret_cast<const char*>(chunk_address_part2),
second_chunk_size);

if (second_chunk_size > 0) {
void* chunk_address_part2 = GetChunk(chunk_number + 1);
std::memcpy(reinterpret_cast<char*>(buffer) + first_chunk_size,
reinterpret_cast<const char*>(chunk_address_part2), second_chunk_size);
}
}

/**
@@ -231,6 +233,8 @@ class MemoryMapManager final {

size_t GetChunkSize() const { return chunk_size_; }

size_t GetNumberOfChunks() const { return number_of_chunks_; }

private:
struct mapping {
boost::interprocess::file_mapping* mapping_;
@@ -118,10 +118,18 @@ struct RawPointerForCompare final {
}

// we do not know the length, first get it, then compare
char buf[8];
persistence_->GetBuffer(l.GetOffset(), buf, 8);
if (persistence_->GetAddressQuickTestOk(l.GetOffset(), 8)) {
length_l = keyvi::util::decodeVarint(reinterpret_cast<uint8_t*>(persistence_->GetAddress(l.GetOffset())));
} else {
char buf[8];
persistence_->GetBuffer(l.GetOffset(), buf, 8);

length_l = keyvi::util::decodeVarint(reinterpret_cast<uint8_t*>(buf));
length_l = keyvi::util::decodeVarint(reinterpret_cast<uint8_t*>(buf));
}

if (length_l != value_size_) {
return false;
}

TRACE("check equality, 3rd buffer %d %d", l.GetOffset(), value_size_);
return persistence_->Compare(l.GetOffset() + keyvi::util::getVarintLength(length_l),
@@ -54,21 +54,50 @@ BOOST_AUTO_TEST_CASE(minimization) {
BOOST_CHECK_EQUAL(w, strings.AddValue("othervalue", &no_minimization));
}

BOOST_AUTO_TEST_CASE(minimization_longvalues) {
JsonValueStore strings(keyvi::util::parameters_t{{TEMPORARY_PATH_KEY, "/tmp"}, {"memory_limit_mb", "10"}});
BOOST_AUTO_TEST_CASE(minimization_largevalues) {
JsonValueStore values(keyvi::util::parameters_t{{TEMPORARY_PATH_KEY, "/tmp"}, {"memory_limit_mb", "10"}});
bool no_minimization = false;
// create a value that is longer than a ushort can store
std::string value = "{\"";
value += std::string(60000, 'a');
value += std::string(70000, 'a');
value += "\":42}";

uint32_t v = strings.AddValue(value, &no_minimization);
uint32_t v = values.AddValue(value, &no_minimization);
BOOST_CHECK_EQUAL(v, 0);
uint32_t w = strings.AddValue("othervalue", &no_minimization);
uint32_t w = values.AddValue("othervalue", &no_minimization);
BOOST_CHECK(v != w);

BOOST_CHECK(w > 0);
BOOST_CHECK_EQUAL(v, strings.AddValue(value, &no_minimization));
BOOST_CHECK_EQUAL(w, strings.AddValue("othervalue", &no_minimization));
BOOST_CHECK_EQUAL(v, values.AddValue(value, &no_minimization));
BOOST_CHECK_EQUAL(w, values.AddValue("othervalue", &no_minimization));
}

BOOST_AUTO_TEST_CASE(minimization_largevalue_small_memory) {
// combination of small value store and large values, spawning more than 1 chunk
JsonValueStore values(keyvi::util::parameters_t{{TEMPORARY_PATH_KEY, "/tmp"}, {"memory_limit", "50000"}});
bool no_minimization = false;
// create a value that almost fills the 1st chunk
std::string padding_value = "{\"";
padding_value += std::string(49990, 'a');
padding_value += "\":33}";
values.AddValue(padding_value, &no_minimization);

std::string value = "{\"";
value += std::string(60000, 'a');
value += "\":42, ";
value += std::string(30000, 'b');
value += "\":99,";
value += std::string(4000, 'c');
value += "\":12}";

uint32_t v = values.AddValue(value, &no_minimization);
BOOST_CHECK_EQUAL(v, 49999);
uint32_t w = values.AddValue("othervalue", &no_minimization);
BOOST_CHECK(v != w);

BOOST_CHECK(w > 0);
BOOST_CHECK_EQUAL(v, values.AddValue(value, &no_minimization));
BOOST_CHECK_EQUAL(w, values.AddValue("othervalue", &no_minimization));
}

BOOST_AUTO_TEST_CASE(minimization2) {
@@ -113,6 +113,22 @@ BOOST_AUTO_TEST_CASE(GetBuffer) {
boost::filesystem::remove_all(path);
}

BOOST_AUTO_TEST_CASE(GetBufferAndNumberOfChunks) {
size_t chunkSize = 1024;

boost::filesystem::path path = boost::filesystem::temp_directory_path();
path /= boost::filesystem::unique_path("dictionary-fsa-unittest-%%%%-%%%%-%%%%-%%%%");
boost::filesystem::create_directory(path);
MemoryMapManager m(chunkSize, path, "basic test");

char buf[8];
m.GetBuffer(10, buf, 8);

BOOST_CHECK_EQUAL(1, m.GetNumberOfChunks());

boost::filesystem::remove_all(path);
}

BOOST_AUTO_TEST_CASE(AppendLargeChunk) {
size_t chunkSize = 4096;

0 comments on commit 10b6717

Please sign in to comment.
You can’t perform that action at this time.