Compressor: Add Brotli Compressor

Signed-off-by: BI SHUN KE <aionshun@livemail.tw>

CMakeLists.txt

@@ -293,6 +293,11 @@ endif(WITH_LEVELDB)
 
 find_package(snappy REQUIRED)
 
+option(WITH_BROTLI "Brotli compression support" OFF)
+if(WITH_BROTLI)
+  set(HAVE_BROTLI TRUE)
+endif()
+
 option(WITH_LZ4 "LZ4 compression support" OFF)
 if(WITH_LZ4)
   find_package(LZ4 1.7 REQUIRED)

src/compressor/CMakeLists.txt

@@ -11,19 +11,27 @@ add_subdirectory(snappy)
 add_subdirectory(zlib)
 add_subdirectory(zstd)
 
-if (HAVE_LZ4)
+if(HAVE_LZ4)
   add_subdirectory(lz4)
 endif()
 
+if(HAVE_BROTLI)
+  add_subdirectory(brotli)
+endif()
+
 set(ceph_compressor_libs
     ceph_snappy
     ceph_zlib
     ceph_zstd)
 
-if (HAVE_LZ4)
+if(HAVE_LZ4)
   list(APPEND ceph_compressor_libs ceph_lz4)
 endif()
 
+if(HAVE_BROTLI)
+  list(APPEND ceph_compressor_libs ceph_brotli)
+endif()
+
 add_custom_target(compressor_plugins DEPENDS
     ${ceph_compressor_libs})
 
@@ -36,8 +44,11 @@ if(WITH_EMBEDDED)
       cephd_compressor_snappy
       cephd_compressor_zlib
       cephd_compressor_zstd)
-  if (HAVE_LZ4)
+  if(HAVE_LZ4)
     list(APPEND cephd_compressor_libs cephd_compressor_lz4)
   endif()
+  if(HAVE_BROTLI)
+    list(APPEND cephd_compressor_libs cephd_compressor_brotli)
+  endif()
   merge_static_libraries(cephd_compressor ${cephd_compressor_libs})
 endif()

src/compressor/Compressor.cc

@@ -30,6 +30,9 @@ const char * Compressor::get_comp_alg_name(int a) {
   case COMP_ALG_ZSTD: return "zstd";
 #ifdef HAVE_LZ4
   case COMP_ALG_LZ4: return "lz4";
+#endif
+#ifdef HAVE_BROTLI
+  case COMP_ALG_BROTLI: return "brotli";
 #endif
   default: return "???";
   }
@@ -45,6 +48,10 @@ boost::optional<Compressor::CompressionAlgorithm> Compressor::get_comp_alg_type(
 #ifdef HAVE_LZ4
   if (s == "lz4")
     return COMP_ALG_LZ4;
+#endif
+#ifdef HAVE_BROTLI
+  if (s == "brotli")
+    return COMP_ALG_BROTLI;
 #endif
   if (s == "" || s == "none")
     return COMP_ALG_NONE;

src/compressor/Compressor.h

@@ -36,6 +36,9 @@ class Compressor {
     COMP_ALG_ZSTD = 3,
 #ifdef HAVE_LZ4
     COMP_ALG_LZ4 = 4,
+#endif
+#ifdef HAVE_BROTLI
+    COMP_ALG_BROTLI = 5,
 #endif
     COMP_ALG_LAST	//the last value for range checks
   };

src/compressor/brotli/BrotliCompressor.cc

@@ -0,0 +1,86 @@
+#include "brotli/encode.h"
+#include "brotli/decode.h"
+#include "BrotliCompressor.h"
+#include "include/scope_guard.h"
+
+#define MAX_LEN (CEPH_PAGE_SIZE)
+
+int BrotliCompressor::compress(const bufferlist &in, bufferlist &out) 
+{
+  BrotliEncoderState* s = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
+  auto sg = make_scope_guard([&s] { BrotliEncoderDestroyInstance(s); });
+  size_t available_in = 0;
+  const uint8_t* next_in = nullptr;
+  size_t available_out = 0;
+  uint8_t* next_out = nullptr;
+  if (!s) {
+      return -1;
+  }
+  BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)9);
+  BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, 22);
+  unsigned char* c_in;
+  for (auto i = in.buffers().begin(); i != in.buffers().end();) {
+    c_in = (unsigned char*) i->c_str();
+    size_t len = i->length();
+    available_in = len;
+    next_in = c_in;
+    ++i;
+    BrotliEncoderOperation finish = i != in.buffers().end() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH;
+    do {
+      size_t max_comp_size = BrotliEncoderMaxCompressedSize(available_in);
+      bufferptr ptr = buffer::create_page_aligned(max_comp_size);
+      next_out = (unsigned char*)ptr.c_str();
+      available_out = max_comp_size;
+      if (!BrotliEncoderCompressStream(s,finish,&available_in, &next_in, &available_out, &next_out, nullptr)) {
+          return -1;
+      }
+      unsigned have = max_comp_size - available_out;
+      out.append(ptr, 0, have);
+    } while (available_out == 0);
+    if (BrotliEncoderIsFinished(s)) {
+      break;
+    }
+  }
+  return 0;
+}
+
+int BrotliCompressor::decompress(bufferlist::iterator &p, size_t compressed_size, bufferlist &out) 
+{
+  const char* c_in;
+  size_t available_in = 0;
+  const uint8_t* next_in = nullptr;
+  size_t available_out = 0;
+  uint8_t* next_out = nullptr;
+  BrotliDecoderState* s = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
+  auto sg = make_scope_guard([&s] { BrotliDecoderDestroyInstance(s); });
+  size_t remaining = std::min<size_t>(p.get_remaining(), compressed_size);
+  if (!s) {
+    return -1;
+  }
+  while (remaining) {
+    size_t len = p.get_ptr_and_advance(remaining, &c_in);
+    remaining -= len;
+    available_in = len;
+    next_in = (unsigned char*)c_in;
+    do {
+      available_out = MAX_LEN;
+      bufferptr ptr = buffer::create_page_aligned(MAX_LEN);
+      next_out = (unsigned char*)ptr.c_str();
+      if (!BrotliDecoderDecompressStream(s, &available_in, &next_in, &available_out, &next_out, 0)) {
+        return -1;
+      }
+      unsigned have = MAX_LEN - available_out;
+      out.append(ptr, 0, have);
+    } while (available_out == 0);
+    if (BrotliDecoderIsFinished(s)) {
+      break;
+    }
+  }
+  return 0;
+}
+
+int BrotliCompressor::decompress(const bufferlist &in, bufferlist &out) 
+{  
+  bufferlist::iterator i = const_cast<bufferlist&>(in).begin();
+  return decompress(i, in.length(), out);
+}

src/compressor/brotli/BrotliCompressor.h

@@ -0,0 +1,31 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 BI SHUN KE <aionshun@livemail.tw>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_BROTLICOMPRESSOR_H
+#define CEPH_BROTLICOMPRESSOR_H
+
+
+#include "include/buffer.h"
+#include "compressor/Compressor.h"
+
+class BrotliCompressor : public Compressor 
+{
+  public:
+  BrotliCompressor() : Compressor(COMP_ALG_BROTLI, "brotli") {}
+
+  int compress(const bufferlist &in, bufferlist &out) override;
+  int decompress(const bufferlist &in, bufferlist &out) override;
+  int decompress(bufferlist::iterator &p, size_t compressed_len, bufferlist &out) override;
+};
+
+#endif //CEPH_BROTLICOMPRESSOR_H
+

src/compressor/brotli/CMakeLists.txt

@@ -0,0 +1,45 @@
+# brotli
+
+set(brotli_sources 
+  CompressionPluginBrotli.cc
+  BrotliCompressor.cc
+)
+include(ExternalProject)
+ExternalProject_Add(brotli_ext
+  DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/src/
+  GIT_REPOSITORY "https://github.com/google/brotli.git"
+  GIT_TAG "master"
+  SOURCE_DIR ${CMAKE_BINARY_DIR}/src/brotli
+  CONFIGURE_COMMAND ./configure-cmake --disable-debug 
+  INSTALL_COMMAND ""
+  BUILD_COMMAND $(MAKE)
+  BUILD_IN_SOURCE 1
+  INSTALL_COMMAND "")
+
+ExternalProject_Add_Step(brotli_ext forcebuild
+  DEPENDEES configure
+  DEPENDERS build
+  COMMAND "true"
+  ALWAYS 1)
+
+set(bortli_libs brotli::common brotli::dec brotli::enc)
+foreach(lib ${bortli_libs})
+  add_library(${lib} STATIC IMPORTED)
+  add_dependencies(${lib} brotli_ext)
+  STRING(REGEX REPLACE "::" "" static_name ${lib})
+  set_property(TARGET ${lib} PROPERTY IMPORTED_LOCATION "${CMAKE_BINARY_DIR}/src/brotli/lib${static_name}-static.a")
+endforeach()
+
+add_library(ceph_brotli SHARED ${brotli_sources})
+add_dependencies(ceph_brotli ${CMAKE_SOURCE_DIR}/src/ceph_ver.h)
+target_include_directories(ceph_brotli	PRIVATE "${CMAKE_BINARY_DIR}/src/brotli/c/include")
+List(REVERSE bortli_libs)
+target_link_libraries(ceph_brotli ${bortli_libs})
+set_target_properties(ceph_brotli PROPERTIES VERSION 2.0.0 SOVERSION 2)
+install(TARGETS ceph_brotli DESTINATION ${compressor_plugin_dir})
+
+if(WITH_EMBEDDED)
+  add_library(cephd_compressor_brotli STATIC ${brotli_sources})
+  target_include_directories(cephd_compressor_brotli  PRIVATE "${CMAKE_BINARY_DIR}/src/brotli/c/include")
+  set_target_properties(cephd_compressor_brotli PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED)
+endif()

src/compressor/brotli/CompressionPluginBrotli.cc

@@ -0,0 +1,18 @@
+#include "acconfig.h"
+#include "ceph_ver.h"
+#include "CompressionPluginBrotli.h"
+
+#ifndef BUILDING_FOR_EMBEDDED
+
+const char *__ceph_plugin_version()
+{
+  return CEPH_GIT_NICE_VER;
+}
+
+int __ceph_plugin_init(CephContext *cct, const std::string& type, const std::string& name)
+{
+  PluginRegistry *instance = cct->get_plugin_registry();
+  return instance->add(type, name, new CompressionPluginBrotli(cct));
+}
+
+#endif // !BUILDING_FOR_EMBEDDED

src/compressor/brotli/CompressionPluginBrotli.h

@@ -0,0 +1,36 @@
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2017 BI SHUN KE <aionshun@livemail.tw>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_COMPRESSION_PLUGIN_BROTLI_H
+#define CEPH_COMPRESSION_PLUGIN_BROTLI_H
+
+#include "ceph_ver.h"
+#include "compressor/CompressionPlugin.h"
+#include "BrotliCompressor.h"
+
+class CompressionPluginBrotli : public CompressionPlugin {
+public:
+  explicit CompressionPluginBrotli(CephContext *cct) : CompressionPlugin(cct)
+  {}
+
+  virtual int factory(CompressorRef *cs, std::ostream *ss)
+  {
+    if (compressor == 0) {
+      BrotliCompressor *interface = new BrotliCompressor();
+      compressor = CompressorRef(interface);
+    }
+    *cs = compressor;
+    return 0;
+  }
+};
+
+#endif

src/include/config-h.in.cmake

@@ -87,6 +87,9 @@
 /* Defined if you have LZ4 */
 #cmakedefine HAVE_LZ4
 
+/* Defined if you have BROTLI */
+#cmakedefine HAVE_BROTLI
+
 /* Defined if you have libaio */
 #cmakedefine HAVE_LIBAIO
 

src/test/compressor/test_compression.cc

@@ -330,7 +330,11 @@ INSTANTIATE_TEST_CASE_P(
 #endif
     "zlib/noisal",
     "snappy",
-    "zstd"));
+#ifdef HAVE_BROTLI
+    "brotli",
+#endif
+    "zstd"
+    ));
 
 #ifdef __x86_64__