From 26d7d3b1fae5a960717ce2b296e4eed177fd9e74 Mon Sep 17 00:00:00 2001 From: Pradeep Rao Date: Fri, 20 May 2022 18:41:26 +0000 Subject: [PATCH] decompressors: stop decompressing upon excessive compression ratio (#733) commit 618d2c7f79d201c508555bcbdaca5538a6e94824 Author: Pradeep Rao Date: Fri May 20 18:09:49 2022 +0000 Fix format. Signed-off-by: Pradeep Rao commit e57c2fc29690522ee571c27c2309612fcfa60c5a Author: Pradeep Rao Date: Thu May 19 18:56:28 2022 +0000 Fix release note. Signed-off-by: Pradeep Rao commit 7a89437589334d58cc0a80d3104714c921b9d9f0 Author: Pradeep Rao Date: Mon May 16 18:16:51 2022 +0000 Fix format. Signed-off-by: Pradeep Rao commit a0abc80be4f180f8a26bb3ec10cd4500a31dc5d1 Author: Pradeep Rao Date: Thu May 12 17:10:56 2022 +0000 decompressors: stop decompressing upon excessive compression ratio (#733) Signed-off-by: Pradeep Rao Signed-off-by: Pradeep Rao --- source/common/runtime/runtime_features.cc | 1 + .../compression/brotli/common/base.cc | 7 ++--- .../compression/brotli/common/base.h | 3 ++- .../compression/brotli/decompressor/BUILD | 1 + .../decompressor/brotli_decompressor_impl.cc | 21 ++++++++++++++- .../extensions/compression/gzip/common/base.h | 1 - .../compression/gzip/decompressor/BUILD | 1 + .../decompressor/zlib_decompressor_impl.cc | 26 ++++++++++++++++++- .../brotli_decompressor_impl_test.cc | 26 +++++++++++++++++++ .../compression/gzip/compressor_fuzz_test.cc | 6 +++-- .../zlib_decompressor_impl_test.cc | 25 ++++++++++++++++++ 11 files changed, 109 insertions(+), 9 deletions(-) diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 939559fd9823..64969356b1e2 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -66,6 +66,7 @@ constexpr const char* runtime_features[] = { "envoy.reloadable_features.check_ocsp_policy", "envoy.reloadable_features.disable_tls_inspector_injection", "envoy.reloadable_features.dont_add_content_length_for_bodiless_requests", + "envoy.reloadable_features.enable_compression_bomb_protection", "envoy.reloadable_features.enable_compression_without_content_length_header", "envoy.reloadable_features.grpc_web_fix_non_proto_encoded_response_handling", "envoy.reloadable_features.grpc_json_transcoder_adhere_to_buffer_limits", diff --git a/source/extensions/compression/brotli/common/base.cc b/source/extensions/compression/brotli/common/base.cc index 12a9a944c8fd..31f05066282f 100644 --- a/source/extensions/compression/brotli/common/base.cc +++ b/source/extensions/compression/brotli/common/base.cc @@ -6,9 +6,10 @@ namespace Compression { namespace Brotli { namespace Common { -BrotliContext::BrotliContext(const uint32_t chunk_size) - : chunk_size_{chunk_size}, chunk_ptr_{std::make_unique(chunk_size)}, next_in_{}, - next_out_{chunk_ptr_.get()}, avail_in_{0}, avail_out_{chunk_size} {} +BrotliContext::BrotliContext(uint32_t chunk_size, uint32_t max_output_size) + : max_output_size_{max_output_size}, chunk_size_{chunk_size}, + chunk_ptr_{std::make_unique(chunk_size)}, next_in_{}, next_out_{chunk_ptr_.get()}, + avail_in_{0}, avail_out_{chunk_size} {} void BrotliContext::updateOutput(Buffer::Instance& output_buffer) { if (avail_out_ == 0) { diff --git a/source/extensions/compression/brotli/common/base.h b/source/extensions/compression/brotli/common/base.h index fed019c9a297..929081c6ef7f 100644 --- a/source/extensions/compression/brotli/common/base.h +++ b/source/extensions/compression/brotli/common/base.h @@ -12,11 +12,12 @@ namespace Common { // Keeps a `Brotli` compression stream's state. struct BrotliContext { - BrotliContext(const uint32_t chunk_size); + BrotliContext(uint32_t chunk_size, uint32_t max_output_size = 0); void updateOutput(Buffer::Instance& output_buffer); void finalizeOutput(Buffer::Instance& output_buffer); + const uint32_t max_output_size_; const uint32_t chunk_size_; std::unique_ptr chunk_ptr_; const uint8_t* next_in_; diff --git a/source/extensions/compression/brotli/decompressor/BUILD b/source/extensions/compression/brotli/decompressor/BUILD index 3667300a8392..f7028b62e02c 100644 --- a/source/extensions/compression/brotli/decompressor/BUILD +++ b/source/extensions/compression/brotli/decompressor/BUILD @@ -19,6 +19,7 @@ envoy_cc_library( "//include/envoy/stats:stats_interface", "//include/envoy/stats:stats_macros", "//source/common/buffer:buffer_lib", + "//source/common/runtime:runtime_features_lib", "//source/extensions/compression/brotli/common:brotli_base_lib", ], ) diff --git a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc index 634c832b8033..a94b6f912682 100644 --- a/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc +++ b/source/extensions/compression/brotli/decompressor/brotli_decompressor_impl.cc @@ -2,12 +2,24 @@ #include +#include "source/common/runtime/runtime_features.h" + namespace Envoy { namespace Extensions { namespace Compression { namespace Brotli { namespace Decompressor { +namespace { + +// How many times the output buffer is allowed to be bigger than the input +// buffer. This value is used to detect compression bombs. +// TODO(rojkov): Re-design the Decompressor interface to handle compression +// bombs gracefully instead of this quick solution. +constexpr uint32_t MaxInflateRatio = 100; + +} // namespace + BrotliDecompressorImpl::BrotliDecompressorImpl(Stats::Scope& scope, const std::string& stats_prefix, const uint32_t chunk_size, const bool disable_ring_buffer_reallocation) @@ -22,7 +34,7 @@ BrotliDecompressorImpl::BrotliDecompressorImpl(Stats::Scope& scope, const std::s void BrotliDecompressorImpl::decompress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) { - Common::BrotliContext ctx(chunk_size_); + Common::BrotliContext ctx(chunk_size_, MaxInflateRatio * input_buffer.length()); for (const Buffer::RawSlice& input_slice : input_buffer.getRawSlices()) { ctx.avail_in_ = input_slice.len_; @@ -58,6 +70,13 @@ bool BrotliDecompressorImpl::process(Common::BrotliContext& ctx, Buffer::Instanc return false; } + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.enable_compression_bomb_protection") && + (output_buffer.length() > ctx.max_output_size_)) { + stats_.brotli_error_.inc(); + return false; + } + ctx.updateOutput(output_buffer); return true; diff --git a/source/extensions/compression/gzip/common/base.h b/source/extensions/compression/gzip/common/base.h index f8b89cb25335..4f427fb90985 100644 --- a/source/extensions/compression/gzip/common/base.h +++ b/source/extensions/compression/gzip/common/base.h @@ -12,7 +12,6 @@ namespace Zlib { /** * Shared code between the compressor and the decompressor. */ -// TODO(junr03): move to extensions tree once the compressor side is moved to extensions. class Base { public: Base(uint64_t chunk_size, std::function zstream_deleter); diff --git a/source/extensions/compression/gzip/decompressor/BUILD b/source/extensions/compression/gzip/decompressor/BUILD index 0a1d8766031b..deb38450eda9 100644 --- a/source/extensions/compression/gzip/decompressor/BUILD +++ b/source/extensions/compression/gzip/decompressor/BUILD @@ -21,6 +21,7 @@ envoy_cc_library( "//source/common/buffer:buffer_lib", "//source/common/common:assert_lib", "//source/common/common:minimal_logger_lib", + "//source/common/runtime:runtime_features_lib", "//source/extensions/compression/gzip/common:zlib_base_lib", ], ) diff --git a/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc b/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc index 9066af8f0426..4c634d00ea04 100644 --- a/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc +++ b/source/extensions/compression/gzip/decompressor/zlib_decompressor_impl.cc @@ -6,7 +6,8 @@ #include "envoy/common/exception.h" -#include "common/common/assert.h" +#include "source/common/common/assert.h" +#include "source/common/runtime/runtime_features.h" #include "absl/container/fixed_array.h" @@ -16,6 +17,16 @@ namespace Compression { namespace Gzip { namespace Decompressor { +namespace { + +// How many times the output buffer is allowed to be bigger than the size of +// accumulated input. This value is used to detect compression bombs. +// TODO(rojkov): Re-design the Decompressor interface to handle compression +// bombs gracefully instead of this quick solution. +constexpr uint64_t MaxInflateRatio = 100; + +} // namespace + ZlibDecompressorImpl::ZlibDecompressorImpl(Stats::Scope& scope, const std::string& stats_prefix) : ZlibDecompressorImpl(scope, stats_prefix, 4096) {} @@ -43,6 +54,8 @@ void ZlibDecompressorImpl::init(int64_t window_bits) { void ZlibDecompressorImpl::decompress(const Buffer::Instance& input_buffer, Buffer::Instance& output_buffer) { + uint64_t limit = MaxInflateRatio * input_buffer.length(); + for (const Buffer::RawSlice& input_slice : input_buffer.getRawSlices()) { zstream_ptr_->avail_in = input_slice.len_; zstream_ptr_->next_in = static_cast(input_slice.mem_); @@ -50,6 +63,17 @@ void ZlibDecompressorImpl::decompress(const Buffer::Instance& input_buffer, if (zstream_ptr_->avail_out == 0) { updateOutput(output_buffer); } + + if (Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.enable_compression_bomb_protection") && + (output_buffer.length() > limit)) { + stats_.zlib_data_error_.inc(); + ENVOY_LOG(trace, + "excessive decompression ratio detected: output " + "size {} for input size {}", + output_buffer.length(), input_buffer.length()); + return; + } } } diff --git a/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc b/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc index cd5d3ee1fd2a..5ab0eaff3381 100644 --- a/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc +++ b/test/extensions/compression/brotli/decompressor/brotli_decompressor_impl_test.cc @@ -26,6 +26,32 @@ class BrotliDecompressorImplTest : public testing::Test { static constexpr uint32_t default_input_size{796}; }; +// Detect excessive compression ratio by compressing a long whitespace string +// into a very small chunk of data and decompressing it again. +TEST_F(BrotliDecompressorImplTest, DetectExcessiveCompressionRatio) { + const absl::string_view ten_whitespaces = " "; + Brotli::Compressor::BrotliCompressorImpl compressor{ + default_quality, + default_window_bits, + default_input_block_bits, + false, + Brotli::Compressor::BrotliCompressorImpl::EncoderMode::Default, + 4096}; + Buffer::OwnedImpl buffer; + + for (int i = 0; i < 1000; i++) { + buffer.add(ten_whitespaces); + } + + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + + Buffer::OwnedImpl output_buffer; + Stats::IsolatedStoreImpl stats_store{}; + BrotliDecompressorImpl decompressor{stats_store, "test.", 16, false}; + decompressor.decompress(buffer, output_buffer); + EXPECT_EQ(1, stats_store.counterFromString("test.brotli_error").value()); +} + // Exercises compression and decompression by compressing some data, decompressing it and then // comparing compressor's input/checksum with decompressor's output/checksum. TEST_F(BrotliDecompressorImplTest, CompressAndDecompress) { diff --git a/test/extensions/compression/gzip/compressor_fuzz_test.cc b/test/extensions/compression/gzip/compressor_fuzz_test.cc index bdaa5283e53a..9a31b865753d 100644 --- a/test/extensions/compression/gzip/compressor_fuzz_test.cc +++ b/test/extensions/compression/gzip/compressor_fuzz_test.cc @@ -72,8 +72,10 @@ DEFINE_FUZZER(const uint8_t* buf, size_t len) { : Envoy::Compression::Compressor::State::Flush); decompressor.decompress(buffer, full_output); } - RELEASE_ASSERT(full_input.toString() == full_output.toString(), ""); - RELEASE_ASSERT(compressor.checksum() == decompressor.checksum(), ""); + if (stats_store.counterFromString("test.zlib_data_error").value() == 0) { + RELEASE_ASSERT(full_input.toString() == full_output.toString(), ""); + RELEASE_ASSERT(compressor.checksum() == decompressor.checksum(), ""); + } } } // namespace Fuzz diff --git a/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc b/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc index 43ae89d42fd0..0e77ffef2c50 100644 --- a/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc +++ b/test/extensions/compression/gzip/decompressor/zlib_decompressor_impl_test.cc @@ -123,6 +123,31 @@ TEST_F(ZlibDecompressorImplTest, CallingChecksum) { ASSERT_EQ(0, decompressor.decompression_error_); } +// Detect excessive compression ratio by compressing a long whitespace string +// into a very small chunk of data and decompressing it again. +TEST_F(ZlibDecompressorImplTest, DetectExcessiveCompressionRatio) { + const absl::string_view ten_whitespaces = " "; + Buffer::OwnedImpl buffer; + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl compressor; + compressor.init( + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionLevel::Standard, + Extensions::Compression::Gzip::Compressor::ZlibCompressorImpl::CompressionStrategy::Standard, + gzip_window_bits, memory_level); + + for (int i = 0; i < 1000; i++) { + buffer.add(ten_whitespaces); + } + + compressor.compress(buffer, Envoy::Compression::Compressor::State::Finish); + + Buffer::OwnedImpl output_buffer; + Stats::IsolatedStoreImpl stats_store{}; + ZlibDecompressorImpl decompressor{stats_store, "test."}; + decompressor.init(gzip_window_bits); + decompressor.decompress(buffer, output_buffer); + ASSERT_EQ(stats_store.counterFromString("test.zlib_data_error").value(), 1); +} + // Exercises compression and decompression by compressing some data, decompressing it and then // comparing compressor's input/checksum with decompressor's output/checksum. TEST_F(ZlibDecompressorImplTest, CompressAndDecompress) {