Permalink
Browse files

SERVER-38168 Vendor Zstandard 1.3.7 to third_party

Added a Zstd based message compressor.
  • Loading branch information...
henrikedin committed Dec 5, 2018
1 parent bc035c2 commit d525ae91e64209267fdaedc17a2542f61b638849
Showing 417 changed files with 149,907 additions and 4 deletions.
@@ -356,6 +356,11 @@ add_option('use-system-zlib',
nargs=0,
)

add_option('use-system-zstd',
help="use system version of Zstandard library",
nargs=0,
)

add_option('use-system-sqlite',
help='use system version of sqlite library',
nargs=0,
@@ -3198,6 +3203,9 @@ def doConfigure(myenv):
if use_system_version_of_library("zlib"):
conf.FindSysLibDep("zlib", ["zdll" if conf.env.TargetOSIs('windows') else "z"])

if use_system_version_of_library("zstd"):
conf.FindSysLibDep("zstd", ["libzstd" if conf.env.TargetOSIs('windows') else "zstd"])

if use_system_version_of_library("stemmer"):
conf.FindSysLibDep("stemmer", ["stemmer"])

@@ -1418,4 +1418,38 @@ THE SOFTWARE.
See the License for the specific language governing permissions and
limitations under the License.

22) License notice for Zstandard
----------------------------

BSD License

For Zstandard software

Copyright (c) 2016-present, Facebook, Inc. All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

* Neither the name Facebook nor the names of its contributors may be used to
endorse or promote products derived from this software without specific
prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

End
@@ -208,7 +208,7 @@ env.CppUnitTest(
)

zlibEnv = env.Clone()
zlibEnv.InjectThirdPartyIncludePaths(libraries=['zlib', 'snappy'])
zlibEnv.InjectThirdPartyIncludePaths(libraries=['zlib', 'zstd', 'snappy'])
zlibEnv.Library(
target='message_compressor',
source=[
@@ -217,12 +217,14 @@ zlibEnv.Library(
'message_compressor_registry.cpp',
'message_compressor_snappy.cpp',
'message_compressor_zlib.cpp',
'message_compressor_zstd.cpp',
],
LIBDEPS=[
'$BUILD_DIR/mongo/base',
'$BUILD_DIR/mongo/util/options_parser/options_parser',
'$BUILD_DIR/third_party/shim_snappy',
'$BUILD_DIR/third_party/shim_zlib',
'$BUILD_DIR/third_party/shim_zstd',
]
)

@@ -42,6 +42,7 @@ enum class MessageCompressor : uint8_t {
kNoop = 0,
kSnappy = 1,
kZlib = 2,
kZstd = 3,
kExtended = 255,
};

@@ -55,7 +56,7 @@ class MessageCompressorBase {
virtual ~MessageCompressorBase() = default;

/*
* Returns the name for subclass compressors (e.g. "snappy", "zlib", or "noop")
* Returns the name for subclass compressors (e.g. "snappy", "zlib", "zstd" or "noop")
*/
const std::string& getName() const {
return _name;
@@ -40,6 +40,7 @@
#include "mongo/transport/message_compressor_registry.h"
#include "mongo/transport/message_compressor_snappy.h"
#include "mongo/transport/message_compressor_zlib.h"
#include "mongo/transport/message_compressor_zstd.h"
#include "mongo/unittest/unittest.h"
#include "mongo/util/log.h"

@@ -235,6 +236,11 @@ TEST(ZlibMessageCompressor, Fidelity) {
checkFidelity(testMessage, stdx::make_unique<ZlibMessageCompressor>());
}

TEST(ZstdMessageCompressor, Fidelity) {
auto testMessage = buildMessage();
checkFidelity(testMessage, stdx::make_unique<ZstdMessageCompressor>());
}

TEST(SnappyMessageCompressor, Overflow) {
checkOverflow(stdx::make_unique<SnappyMessageCompressor>());
}
@@ -243,11 +249,19 @@ TEST(ZlibMessageCompressor, Overflow) {
checkOverflow(stdx::make_unique<ZlibMessageCompressor>());
}

TEST(ZstdMessageCompressor, Overflow) {
checkOverflow(stdx::make_unique<ZstdMessageCompressor>());
}

TEST(MessageCompressorManager, SERVER_28008) {

// Create a client and server that will negotiate the same compressors,
// but with a different ordering for the preferred compressor.

std::unique_ptr<MessageCompressorBase> zstdCompressor =
stdx::make_unique<ZstdMessageCompressor>();
const auto zstdId = zstdCompressor->getId();

std::unique_ptr<MessageCompressorBase> zlibCompressor =
stdx::make_unique<ZlibMessageCompressor>();
const auto zlibId = zlibCompressor->getId();
@@ -257,8 +271,10 @@ TEST(MessageCompressorManager, SERVER_28008) {
const auto snappyId = snappyCompressor->getId();

MessageCompressorRegistry registry;
registry.setSupportedCompressors({snappyCompressor->getName(), zlibCompressor->getName()});
registry.setSupportedCompressors(
{snappyCompressor->getName(), zlibCompressor->getName(), zstdCompressor->getName()});
registry.registerImplementation(std::move(zlibCompressor));
registry.registerImplementation(std::move(zstdCompressor));
registry.registerImplementation(std::move(snappyCompressor));
ASSERT_OK(registry.finalizeSupportedCompressors());

@@ -294,6 +310,17 @@ TEST(MessageCompressorManager, SERVER_28008) {
toSend = assertOk(serverManager.compressMessage(recvd, &compressorId));
recvd = assertOk(clientManager.decompressMessage(toSend, &compressorId));
ASSERT_EQ(compressorId, zlibId);

// Then, force the client to send as zstd. We should round trip as
// zstd if we feed the out compresor id parameter from
// decompressMessage back in to compressMessage.
toSend = buildMessage();
toSend = assertOk(clientManager.compressMessage(toSend, &zstdId));
recvd = assertOk(serverManager.decompressMessage(toSend, &compressorId));
ASSERT_EQ(compressorId, zstdId);
toSend = assertOk(serverManager.compressMessage(recvd, &compressorId));
recvd = assertOk(clientManager.decompressMessage(toSend, &compressorId));
ASSERT_EQ(compressorId, zstdId);
}

TEST(MessageCompressorManager, MessageSizeTooLarge) {
@@ -37,6 +37,7 @@
#include "mongo/transport/message_compressor_noop.h"
#include "mongo/transport/message_compressor_snappy.h"
#include "mongo/transport/message_compressor_zlib.h"
#include "mongo/transport/message_compressor_zstd.h"
#include "mongo/util/options_parser/option_section.h"

#include <boost/algorithm/string/classification.hpp>
@@ -45,7 +46,7 @@
namespace mongo {
namespace {
const auto kDisabledConfigValue = "disabled"_sd;
const auto kDefaultConfigValue = "snappy,zlib"_sd;
const auto kDefaultConfigValue = "snappy,zstd,zlib"_sd;
} // namespace

StringData getMessageCompressorName(MessageCompressor id) {
@@ -56,6 +57,8 @@ StringData getMessageCompressorName(MessageCompressor id) {
return "snappy"_sd;
case MessageCompressor::kZlib:
return "zlib"_sd;
case MessageCompressor::kZstd:
return "zstd"_sd;
default:
fassert(40269, "Invalid message compressor ID");
}
@@ -0,0 +1,89 @@

/**
* Copyright (C) 2018-present MongoDB, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the Server Side Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/

#define MONGO_LOG_DEFAULT_COMPONENT ::mongo::logger::LogComponent::kNetwork

#include "mongo/platform/basic.h"

#include <zstd.h>

#include "mongo/base/init.h"
#include "mongo/stdx/memory.h"
#include "mongo/transport/message_compressor_registry.h"
#include "mongo/transport/message_compressor_zstd.h"

namespace mongo {

ZstdMessageCompressor::ZstdMessageCompressor() : MessageCompressorBase(MessageCompressor::kZstd) {}

std::size_t ZstdMessageCompressor::getMaxCompressedSize(size_t inputSize) {
return ZSTD_compressBound(inputSize);
}

StatusWith<std::size_t> ZstdMessageCompressor::compressData(ConstDataRange input,
DataRange output) {
size_t ret = ZSTD_compress(const_cast<char*>(output.data()),
output.length(),
input.data(),
input.length(),
ZSTD_CLEVEL_DEFAULT);

if (ZSTD_isError(ret)) {
return Status{ErrorCodes::BadValue,
str::stream() << "Could not compress input: " << ZSTD_getErrorName(ret)};
}
counterHitCompress(input.length(), ret);
return {ret};
}

StatusWith<std::size_t> ZstdMessageCompressor::decompressData(ConstDataRange input,
DataRange output) {
size_t ret = ZSTD_decompress(
const_cast<char*>(output.data()), output.length(), input.data(), input.length());

if (ZSTD_isError(ret)) {
return Status{ErrorCodes::BadValue,
str::stream() << "Could not decompress message: " << ZSTD_getErrorName(ret)};
}

counterHitDecompress(input.length(), ret);
return {ret};
}


MONGO_INITIALIZER_GENERAL(ZstdMessageCompressorInit,
("EndStartupOptionHandling"),
("AllCompressorsRegistered"))
(InitializerContext* context) {
auto& compressorRegistry = MessageCompressorRegistry::get();
compressorRegistry.registerImplementation(stdx::make_unique<ZstdMessageCompressor>());
return Status::OK();
}
} // namespace mongo
@@ -0,0 +1,46 @@

/**
* Copyright (C) 2018-present MongoDB, Inc.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the Server Side Public License, version 1,
* as published by MongoDB, Inc.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* Server Side Public License for more details.
*
* You should have received a copy of the Server Side Public License
* along with this program. If not, see
* <http://www.mongodb.com/licensing/server-side-public-license>.
*
* As a special exception, the copyright holders give permission to link the
* code of portions of this program with the OpenSSL library under certain
* conditions as described in each individual source file and distribute
* linked combinations including the program with the OpenSSL library. You
* must comply with the Server Side Public License in all respects for
* all of the code used other than as permitted herein. If you modify file(s)
* with this exception, you may extend this exception to your version of the
* file(s), but you are not obligated to do so. If you do not wish to do so,
* delete this exception statement from your version. If you delete this
* exception statement from all source files in the program, then also delete
* it in the license file.
*/

#include "mongo/transport/message_compressor_base.h"

namespace mongo {
class ZstdMessageCompressor final : public MessageCompressorBase {
public:
ZstdMessageCompressor();

std::size_t getMaxCompressedSize(size_t inputSize) override;

StatusWith<std::size_t> compressData(ConstDataRange input, DataRange output) override;

StatusWith<std::size_t> decompressData(ConstDataRange input, DataRange output) override;
};


} // namespace mongo
@@ -9,6 +9,7 @@ Import("mobile_se")
boostSuffix = "-1.68.0"
snappySuffix = '-1.1.3'
zlibSuffix = '-1.2.11'
zstdSuffix = '-1.3.7'
pcreSuffix = "-8.41"
mozjsSuffix = '-45'
yamlSuffix = '-0.6.2'
@@ -53,6 +54,10 @@ if not use_system_version_of_library('zlib'):
thirdPartyIncludePathList.append(
('zlib', '#/src/third_party/zlib' + zlibSuffix))

if not use_system_version_of_library('zstd'):
thirdPartyIncludePathList.append(
('zstd', '#/src/third_party/zstandard' + zstdSuffix + '/zstd/lib'))

if not use_system_version_of_library('sqlite'):
thirdPartyIncludePathList.append(
('sqlite', '#/src/third_party/sqlite' + sqliteSuffix + '/sqlite'))
@@ -247,6 +252,26 @@ zlibEnv.Library(
'shim_zlib.cpp',
])

if use_system_version_of_library("zstd"):
zstdEnv = env.Clone(
SYSLIBDEPS=[
env['LIBDEPS_ZSTD_SYSLIBDEP'],
])
else:
zstdEnv = env.Clone()
zstdEnv.InjectThirdPartyIncludePaths(libraries=['zstd'])
zstdEnv.SConscript('zstandard' + zstdSuffix + '/SConscript', exports={ 'env' : zstdEnv })
zstdEnv = zstdEnv.Clone(
LIBDEPS_INTERFACE=[
'zstandard' + zstdSuffix + '/zstd',
])

zstdEnv.Library(
target="shim_zstd",
source=[
'shim_zstd.cpp',
])

if use_system_version_of_library("google-benchmark"):
benchmarkEnv = env.Clone(
SYSLIBDEPS=[
Oops, something went wrong.

0 comments on commit d525ae9

Please sign in to comment.