Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
compressor/zstd: add zstd compressor
Build/link of zstd itself is maybe not ideal, but it works fine. Signed-off-by: Sage Weil <sage@redhat.com> Signed-off-by: Kefu Chai <kchai@redhat.com>
- Loading branch information
Showing
7 changed files
with
212 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,42 @@ | ||
# zstd | ||
|
||
# libzstd - build it statically | ||
list(APPEND ZSTD_CMAKE_ARGS | ||
-DCMAKE_C_COMPILER_ARGS="-fPIC -Wno-unused-variable -O3") | ||
|
||
include(ExternalProject) | ||
ExternalProject_Add(zstd_ext | ||
SOURCE_DIR ${CMAKE_SOURCE_DIR}/src/zstd/build/cmake | ||
CMAKE_ARGS ${ZSTD_CMAKE_ARGS} | ||
BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR} | ||
BUILD_COMMAND $(MAKE) libzstd_static | ||
INSTALL_COMMAND "true") | ||
|
||
# force zstd make to be called on each time | ||
ExternalProject_Add_Step(zstd_ext forcebuild | ||
DEPENDEES configure | ||
DEPENDERS build | ||
COMMAND "true" | ||
ALWAYS 1) | ||
|
||
add_library(zstd STATIC IMPORTED) | ||
set_property(TARGET zstd PROPERTY | ||
IMPORTED_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/lib/libzstd.a") | ||
add_dependencies(zstd zstd_ext) | ||
set(ZSTD_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/src/zstd/lib) | ||
|
||
# | ||
set(zstd_sources | ||
CompressionPluginZstd.cc | ||
) | ||
|
||
add_library(ceph_zstd SHARED ${zstd_sources}) | ||
add_dependencies(ceph_zstd ${CMAKE_SOURCE_DIR}/src/ceph_ver.h) | ||
target_link_libraries(ceph_zstd zstd) | ||
set_target_properties(ceph_zstd PROPERTIES VERSION 2.0.0 SOVERSION 2) | ||
install(TARGETS ceph_zstd DESTINATION ${compressor_plugin_dir}) | ||
|
||
if(WITH_EMBEDDED) | ||
add_library(cephd_compressor_zstd STATIC ${zstd_sources}) | ||
set_target_properties(cephd_compressor_zstd PROPERTIES COMPILE_DEFINITIONS BUILDING_FOR_EMBEDDED) | ||
endif() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
/* | ||
* Ceph - scalable distributed file system | ||
* | ||
* Copyright (C) 2015 Mirantis, Inc. | ||
* | ||
* Author: Alyona Kiseleva <akiselyova@mirantis.com> | ||
* | ||
* This library is free software; you can redistribute it and/or | ||
* modify it under the terms of the GNU Lesser General Public | ||
* License as published by the Free Software Foundation; either | ||
* version 2.1 of the License, or (at your option) any later version. | ||
* | ||
*/ | ||
|
||
#include <ostream> | ||
|
||
// ----------------------------------------------------------------------------- | ||
#include "ceph_ver.h" | ||
#include "compressor/CompressionPlugin.h" | ||
#include "ZstdCompressor.h" | ||
// ----------------------------------------------------------------------------- | ||
|
||
class CompressionPluginZstd : public CompressionPlugin { | ||
|
||
public: | ||
|
||
explicit CompressionPluginZstd(CephContext* cct) : CompressionPlugin(cct) | ||
{} | ||
|
||
virtual int factory(CompressorRef *cs, | ||
std::ostream *ss) | ||
{ | ||
if (compressor == 0) { | ||
ZstdCompressor *interface = new ZstdCompressor(); | ||
compressor = CompressorRef(interface); | ||
} | ||
*cs = compressor; | ||
return 0; | ||
} | ||
}; | ||
|
||
// ----------------------------------------------------------------------------- | ||
|
||
const char *__ceph_plugin_version() | ||
{ | ||
return CEPH_GIT_NICE_VER; | ||
} | ||
|
||
// ----------------------------------------------------------------------------- | ||
|
||
int __ceph_plugin_init(CephContext *cct, | ||
const std::string& type, | ||
const std::string& name) | ||
{ | ||
PluginRegistry *instance = cct->get_plugin_registry(); | ||
|
||
return instance->add(type, name, new CompressionPluginZstd(cct)); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,98 @@ | ||
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- | ||
// vim: ts=8 sw=2 smarttab | ||
/* | ||
* Ceph - scalable distributed file system | ||
* | ||
* Copyright (C) 2015 Haomai Wang <haomaiwang@gmail.com> | ||
* | ||
* This is free software; you can redistribute it and/or | ||
* modify it under the terms of the GNU Lesser General Public | ||
* License version 2.1, as published by the Free Software | ||
* Foundation. See file COPYING. | ||
* | ||
*/ | ||
|
||
#ifndef CEPH_ZSTDCOMPRESSOR_H | ||
#define CEPH_ZSTDCOMPRESSOR_H | ||
|
||
#include "zstd/lib/zstd.h" | ||
#include "include/buffer.h" | ||
#include "include/encoding.h" | ||
#include "compressor/Compressor.h" | ||
|
||
#define COMPRESSION_LEVEL 5 | ||
|
||
class ZstdCompressor : public Compressor { | ||
public: | ||
ZstdCompressor() : Compressor(COMP_ALG_ZSTD, "zstd") {} | ||
|
||
int compress(const bufferlist &src, bufferlist &dst) override { | ||
bufferptr outptr = buffer::create_page_aligned( | ||
ZSTD_compressBound(src.length())); | ||
ZSTD_outBuffer_s outbuf; | ||
outbuf.dst = outptr.c_str(); | ||
outbuf.size = outptr.length(); | ||
outbuf.pos = 0; | ||
|
||
ZSTD_CStream *s = ZSTD_createCStream(); | ||
ZSTD_initCStream(s, COMPRESSION_LEVEL); | ||
auto p = src.begin(); | ||
size_t left = src.length(); | ||
while (left) { | ||
assert(!p.end()); | ||
struct ZSTD_inBuffer_s inbuf; | ||
inbuf.pos = 0; | ||
inbuf.size = p.get_ptr_and_advance(left, (const char**)&inbuf.src); | ||
ZSTD_compressStream(s, &outbuf, &inbuf); | ||
left -= inbuf.size; | ||
} | ||
assert(p.end()); | ||
ZSTD_flushStream(s, &outbuf); | ||
ZSTD_endStream(s, &outbuf); | ||
|
||
// prefix with decompressed length | ||
::encode((uint32_t)src.length(), dst); | ||
dst.append(outptr, 0, outbuf.pos); | ||
return 0; | ||
} | ||
|
||
int decompress(const bufferlist &src, bufferlist &dst) override { | ||
bufferlist::iterator i = const_cast<bufferlist&>(src).begin(); | ||
return decompress(i, src.length(), dst); | ||
} | ||
|
||
int decompress(bufferlist::iterator &p, | ||
size_t compressed_len, | ||
bufferlist &dst) override { | ||
if (compressed_len < 4) { | ||
return -1; | ||
} | ||
compressed_len -= 4; | ||
uint32_t dst_len; | ||
::decode(dst_len, p); | ||
|
||
bufferptr dstptr(dst_len); | ||
ZSTD_outBuffer_s outbuf; | ||
outbuf.dst = dstptr.c_str(); | ||
outbuf.size = dstptr.length(); | ||
outbuf.pos = 0; | ||
ZSTD_DStream *s = ZSTD_createDStream(); | ||
ZSTD_initDStream(s); | ||
while (compressed_len > 0) { | ||
if (p.end()) { | ||
return -1; | ||
} | ||
ZSTD_inBuffer_s inbuf; | ||
inbuf.pos = 0; | ||
inbuf.size = p.get_ptr_and_advance(compressed_len, (const char**)&inbuf.src); | ||
ZSTD_decompressStream(s, &outbuf, &inbuf); | ||
compressed_len -= inbuf.size; | ||
} | ||
ZSTD_freeDStream(s); | ||
|
||
dst.append(dstptr, 0, outbuf.pos); | ||
return 0; | ||
} | ||
}; | ||
|
||
#endif |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters