Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add compression of posting lists for inverted index #56226

Merged
merged 1 commit into from Nov 2, 2023
Merged

Add compression of posting lists for inverted index #56226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
30 changes: 23 additions & 7 deletions src/Storages/MergeTree/GinIndexStore.cpp
View file
Open in desktop
@@ -1,6 +1,7 @@
#include <Storages/MergeTree/GinIndexStore.h>
#include <Columns/ColumnString.h>
#include <Common/FST.h>
#include <Compression/CompressionFactory.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
Expand Down Expand Up @@ -81,14 +82,22 @@ UInt64 GinIndexPostingsBuilder::serialize(WriteBuffer & buffer)
{
rowid_bitmap.runOptimize();
auto size = rowid_bitmap.getSizeInBytes();
auto buf = std::make_unique<char[]>(size);
rowid_bitmap.write(buf.get());

auto codec = CompressionCodecFactory::instance().get(GIN_COMPRESSION_CODEC, GIN_COMPRESSION_LEVEL);
Memory<> memory;
memory.resize(codec->getCompressedReserveSize(static_cast<UInt32>(size)));
rschu1ze marked this conversation as resolved.
Show resolved Hide resolved
auto compressed_size = codec->compress(buf.get(), static_cast<UInt32>(size), memory.data());

writeVarUInt(size, buffer);
written_bytes += getLengthOfVarUInt(size);

auto buf = std::make_unique<char[]>(size);
rowid_bitmap.write(buf.get());
buffer.write(buf.get(), size);
written_bytes += size;
writeVarUInt(compressed_size, buffer);
written_bytes += getLengthOfVarUInt(compressed_size);

buffer.write(memory.data(), compressed_size);
written_bytes += compressed_size;
}
else
{
Expand All @@ -110,11 +119,18 @@ GinIndexPostingsListPtr GinIndexPostingsBuilder::deserialize(ReadBuffer & buffer
if (postings_list_size == USES_BIT_MAP)
{
size_t size = 0;
size_t compressed_size = 0;
readVarUInt(size, buffer);
auto buf = std::make_unique<char[]>(size);
buffer.readStrict(reinterpret_cast<char *>(buf.get()), size);
readVarUInt(compressed_size, buffer);
auto buf = std::make_unique<char[]>(compressed_size);
buffer.readStrict(reinterpret_cast<char *>(buf.get()), compressed_size);

Memory<> memory;
memory.resize(size);
auto codec = CompressionCodecFactory::instance().get(GIN_COMPRESSION_CODEC, GIN_COMPRESSION_LEVEL);
codec->decompress(buf.get(), static_cast<UInt32>(compressed_size), memory.data());

GinIndexPostingsListPtr postings_list = std::make_shared<GinIndexPostingsList>(GinIndexPostingsList::read(buf.get()));
GinIndexPostingsListPtr postings_list = std::make_shared<GinIndexPostingsList>(GinIndexPostingsList::read(memory.data()));

return postings_list;
}
Expand Down
5 changes: 4 additions & 1 deletion src/Storages/MergeTree/GinIndexStore.h
View file
Open in desktop
Expand Up @@ -61,6 +61,9 @@ class GinIndexPostingsBuilder
private:
constexpr static int MIN_SIZE_FOR_ROARING_ENCODING = 16;

static constexpr auto GIN_COMPRESSION_CODEC = "ZSTD";
static constexpr auto GIN_COMPRESSION_LEVEL = 1;

/// When the list length is no greater than MIN_SIZE_FOR_ROARING_ENCODING, array 'rowid_lst' is used
/// As a special case, rowid_lst[0] == CONTAINS_ALL encodes that all rowids are set.
std::array<UInt32, MIN_SIZE_FOR_ROARING_ENCODING> rowid_lst;
Expand Down Expand Up @@ -211,7 +214,7 @@ class GinIndexStore
v1 = 1, /// Initial version
};

static constexpr auto CURRENT_GIN_FILE_FORMAT_VERSION = Format::v0;
static constexpr auto CURRENT_GIN_FILE_FORMAT_VERSION = Format::v1;
};

using GinIndexStorePtr = std::shared_ptr<GinIndexStore>;
Expand Down