Added libzstd (compression library) and a simple wrapper for it. (#463)

.github/workflows/cmake.yml

@@ -32,7 +32,7 @@ jobs:
     - uses: actions/checkout@v2
 
     - name: Install dependencies
-      run:  sudo apt-get install -y libsparsehash-dev libicu-dev tzdata gcc-10 libjemalloc-dev
+      run:  sudo apt-get install -y libsparsehash-dev libicu-dev tzdata gcc-10 libzstd-dev libjemalloc-dev
     - name: Install clang 12
       run: wget https://apt.llvm.org/llvm.sh && sudo chmod +x llvm.sh && sudo ./llvm.sh 12
 

Dockerfile

@@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 FROM base as builder
 RUN apt-get update && apt-get install -y build-essential cmake libsparsehash-dev libicu-dev tzdata pkg-config uuid-runtime uuid-dev git
-RUN apt install -y libjemalloc-dev ninja-build
+RUN apt install -y libjemalloc-dev ninja-build libzstd-dev
 
 COPY . /app/
 
@@ -22,7 +22,7 @@ FROM base as runtime
 WORKDIR /app
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y wget python3-yaml unzip curl bzip2 pkg-config libicu-dev python3-icu libgomp1 uuid-runtime
-RUN apt install -y lbzip2 libjemalloc-dev
+RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev
 
 ARG UID=1000
 RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app

src/util/CompressionUsingZstd/ZstdWrapper.h

@@ -0,0 +1,52 @@
+// Copyright 2021, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author: Johannes Kalmbach <johannes.kalmbach@gmail.com>
+
+#pragma once
+
+#include <zstd.h>
+
+#include <vector>
+
+#include "../Exception.h"
+
+class ZstdWrapper {
+ public:
+  // Compress the given byte array and return the result;
+  static std::vector<char> compress(void* src, size_t numBytes,
+                                    int compressionLevel = 3) {
+    std::vector<char> result(ZSTD_compressBound(numBytes));
+    auto compressedSize = ZSTD_compress(result.data(), result.size(), src,
+                                        numBytes, compressionLevel);
+    result.resize(compressedSize);
+    return result;
+  }
+
+  // Decompress the given byte array, assuming that the size of the decompressed
+  // data is known.
+  template <typename T>
+  requires(std::is_trivially_copyable_v<T>) static std::vector<T> decompress(
+      void* src, size_t numBytes, size_t knownOriginalSize) {
+    knownOriginalSize *= sizeof(T);
+    std::vector<T> result(knownOriginalSize / sizeof(T));
+    auto compressedSize =
+        ZSTD_decompress(result.data(), knownOriginalSize, src, numBytes);
+    AD_CHECK(compressedSize == knownOriginalSize);
+    return result;
+  }
+
+  // Decompress the given byte array to the given buffer of the given size,
+  // returning the number of bytes of the decompressed data.
+  template <typename T>
+  requires(std::is_trivially_copyable_v<T>) static size_t
+      decompressToBuffer(const char* src, size_t numBytes, T* buffer,
+                         size_t bufferCapacity) {
+    auto decompressedSize = ZSTD_decompress(buffer, bufferCapacity,
+                                            const_cast<char*>(src), numBytes);
+    if (ZSTD_isError(decompressedSize)) {
+      throw std::runtime_error(std::string("error during decompression : ") +
+                               ZSTD_getErrorName(decompressedSize));
+    }
+    return decompressedSize;
+  }
+};

test/CMakeLists.txt

@@ -174,3 +174,7 @@ add_test(SparqlAntlrParserTest SparqlAntlrParserTest)
 add_executable(SerializerTest SerializerTest.cpp)
 target_link_libraries(SerializerTest gtest_main absl::flat_hash_map ${CMAKE_THREAD_LIBS_INIT})
 add_test(SerializerTest SerializerTest)
+
+add_executable(ZstdCompressionTest ZstdCompressionTest.cpp)
+target_link_libraries(ZstdCompressionTest gtest_main absl::flat_hash_map zstd ${cmake_thread_libs_init})
+add_test(ZstdCompressionTest ZstdCompressionTest )

test/ZstdCompressionTest.cpp

@@ -0,0 +1,28 @@
+// Copyright 2021, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author: Johannes Kalmbach <johannes.kalmbach@gmail.com>
+
+#include <gtest/gtest.h>
+
+#include "../src/util/CompressionUsingZstd/ZstdWrapper.h"
+
+// _____________________________________________________________________________
+TEST(CompressionTest, Basic) {
+  std::vector<int> x{1, 2, 3, 4};
+  std::vector<char> comp =
+      ZstdWrapper::compress(x.data(), x.size() * sizeof(int));
+  auto decomp = ZstdWrapper::decompress<int>(comp.data(), comp.size(), 4);
+  ASSERT_EQ(x, decomp);
+}
+
+// _____________________________________________________________________________
+TEST(CompressionTest, DecompressToBuffer) {
+  std::vector<int> x{1, 2, 3, 4};
+  std::vector<char> comp =
+      ZstdWrapper::compress(x.data(), x.size() * sizeof(int));
+  std::vector<int> decomp(4);
+  auto numBytesDecompressed = ZstdWrapper::decompressToBuffer<int>(
+      comp.data(), comp.size(), decomp.data(), decomp.size() * sizeof(int));
+  ASSERT_EQ(x, decomp);
+  ASSERT_EQ(4ul * sizeof(int), numBytesDecompressed);
+}