From 056ca4f555fbbf4463de5be8642a2c01b6759192 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Wed, 24 May 2023 13:26:15 -0400 Subject: [PATCH 01/20] Add extensive testing cases for deflate qpl codec --- .../sql-reference/statements/create/table.md | 2 +- src/Client/Connection.cpp | 2 +- src/Compression/CompressionCodecDeflateQpl.h | 3 +- src/Compression/CompressionFactory.h | 4 +- .../CompressionFactoryAdditions.cpp | 14 ++-- src/Compression/ICompressionCodec.h | 3 + src/Core/Settings.h | 1 + src/Interpreters/InterpreterCreateQuery.cpp | 3 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/AlterCommands.cpp | 8 +-- src/Storages/ColumnsDescription.cpp | 2 +- src/Storages/Distributed/DistributedSink.cpp | 2 +- src/Storages/TTLDescription.cpp | 2 +- .../deflateqpl_compression_by_default.xml | 11 ++++ .../configs/enable_deflateqpl_codec.xml | 7 ++ .../test_non_default_compression/test.py | 65 ++++++++++++++++++- ...04_test_alter_compression_codecs.reference | 19 +++++- .../00804_test_alter_compression_codecs.sql | 24 +++++-- 18 files changed, 145 insertions(+), 29 deletions(-) create mode 100644 tests/integration/test_non_default_compression/configs/deflateqpl_compression_by_default.xml create mode 100644 tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index de44a0014722..b0865ad28964 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -380,7 +380,7 @@ High compression levels are useful for asymmetric scenarios, like compress once, `DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply: -- DEFLATE_QPL is experimental and can only be used after setting configuration parameter `allow_experimental_codecs=1`. +- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_qpl_deflate=1`. - DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details. - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details. - DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled. diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 2350a5039ab1..68bc3b39a560 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -588,7 +588,7 @@ void Connection::sendQuery( if (method == "ZSTD") level = settings->network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_qpl_deflate); compression_codec = CompressionCodecFactory::instance().get(method, level); } else diff --git a/src/Compression/CompressionCodecDeflateQpl.h b/src/Compression/CompressionCodecDeflateQpl.h index 7a1a764295d8..13aa8733b54a 100644 --- a/src/Compression/CompressionCodecDeflateQpl.h +++ b/src/Compression/CompressionCodecDeflateQpl.h @@ -98,7 +98,8 @@ class CompressionCodecDeflateQpl final : public ICompressionCodec protected: bool isCompression() const override { return true; } bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return true; } + bool isExperimental() const override { return false; } + bool isDeflateQplCompression() const override { return true; } UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index a4451f9ed2e7..1fdaf4f1c713 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -40,10 +40,10 @@ class CompressionCodecFactory final : private boost::noncopyable CompressionCodecPtr getDefaultCodec() const; /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const; + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const; /// Validate codecs AST specified by user - void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const; + void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const; /// Get codec by AST and possible column_type. Some codecs can use /// information about type to improve inner settings, but every codec should diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index 978a0fe5069f..2630326238af 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -34,7 +34,7 @@ namespace ErrorCodes void CompressionCodecFactory::validateCodec( - const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs) const + const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const { if (family_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty"); @@ -43,13 +43,13 @@ void CompressionCodecFactory::validateCodec( { auto literal = std::make_shared(static_cast(*level)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), - {}, sanity_check, allow_experimental_codecs); + {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate); } else { auto identifier = std::make_shared(Poco::toUpper(family_name)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), - {}, sanity_check, allow_experimental_codecs); + {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate); } } @@ -77,7 +77,7 @@ bool innerDataTypeIsFloat(const DataTypePtr & type) } ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( - const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs) const + const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const { if (const auto * func = ast->as()) { @@ -159,6 +159,12 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( " You can enable it with the 'allow_experimental_codecs' setting.", codec_family_name); + if (!enable_qpl_deflate && result_codec->isDeflateQplCompression()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Codec {} is disabled by default." + " You can enable it with the 'enable_qpl_deflate' setting.", + codec_family_name); + codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); } diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index 44835ac19cb1..d92ad3fc718f 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -112,6 +112,9 @@ class ICompressionCodec : private boost::noncopyable /// If it does nothing. virtual bool isNone() const { return false; } + /// This is a knob for Deflate QPL codec. + virtual bool isDeflateQplCompression() const { return false; } + protected: /// This is used for fuzz testing friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 464b9168a4cd..c6a2069e6ae2 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -319,6 +319,7 @@ class IColumn; M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ + M(Bool, enable_qpl_deflate, false, "If it is set to true, allow to use deflate_qpl for compression.", 0) \ M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ab9e1fb04d6d..5c22b46b3607 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -571,6 +571,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; + bool enable_qpl_deflate = attach || context_->getSettingsRef().enable_qpl_deflate; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); @@ -631,7 +632,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.default_specifier == "ALIAS") throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs); + col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_qpl_deflate); } if (col_decl.ttl) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 1ded7d972482..96c585e7d164 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1775,7 +1775,7 @@ void TCPHandler::initBlockOutput(const Block & block) if (state.compression == Protocol::Compression::Enable) { - CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_qpl_deflate); state.maybe_compressed_out = std::make_shared( *out, CompressionCodecFactory::instance().get(method, level)); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 5fd823b9e013..ecbddfc3e2a0 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -388,7 +388,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) column.comment = *comment; if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false, true); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type, false, true, true); column.ttl = ttl; @@ -429,7 +429,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context) else { if (codec) - column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false, true); + column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(codec, data_type ? data_type : column.type, false, true, true); if (comment) column.comment = *comment; @@ -1067,7 +1067,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const "this column name is reserved for lightweight delete feature", backQuote(column_name)); if (command.codec) - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate); all_columns.add(ColumnDescription(column_name, command.data_type)); } @@ -1093,7 +1093,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const { if (all_columns.hasAlias(column_name)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate); } auto column_default = all_columns.getDefault(column_name); if (column_default) diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 8eabae7929c7..045afd7e6e64 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -130,7 +130,7 @@ void ColumnDescription::readText(ReadBuffer & buf) comment = col_ast->comment->as().value.get(); if (col_ast->codec) - codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false, true); + codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(col_ast->codec, type, false, true, true); if (col_ast->ttl) ttl = col_ast->ttl; diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 720a951299a1..ce1dbde8eaec 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -733,7 +733,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const if (compression_method == "ZSTD") compression_level = settings.network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs); + CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_qpl_deflate); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index e1a80800630e..f5209cbdff6a 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -285,7 +285,7 @@ TTLDescription TTLDescription::getTTLFromAST( { result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs); + ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate); } } diff --git a/tests/integration/test_non_default_compression/configs/deflateqpl_compression_by_default.xml b/tests/integration/test_non_default_compression/configs/deflateqpl_compression_by_default.xml new file mode 100644 index 000000000000..2ad6a0f1eff6 --- /dev/null +++ b/tests/integration/test_non_default_compression/configs/deflateqpl_compression_by_default.xml @@ -0,0 +1,11 @@ + + + + + 0 + 0 + + deflate_qpl + + + diff --git a/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml b/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml new file mode 100644 index 000000000000..46e9e43ca276 --- /dev/null +++ b/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml @@ -0,0 +1,7 @@ + + + + 1 + + + diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index e0a67a5db957..e69b32daae08 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -41,7 +41,14 @@ main_configs=["configs/allow_experimental_codecs.xml"], user_configs=["configs/allow_suspicious_codecs.xml"], ) - +node7 = cluster.add_instance( + "node7", + main_configs=["configs/deflateqpl_compression_by_default.xml"], + user_configs=[ + "configs/enable_deflateqpl_codec.xml", + "configs/allow_suspicious_codecs.xml", + ], +) @pytest.fixture(scope="module") def start_cluster(): @@ -244,3 +251,59 @@ def test_uncompressed_cache_plus_zstd_codec(start_cluster): ) == "10000\n" ) + +def test_preconfigured_deflateqpl_codec(start_cluster): + node7.query( + """ + CREATE TABLE compression_codec_multiple_with_key ( + somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), DEFLATE_QPL), + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, DEFLATE_QPL), + data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4, DEFLATE_QPL), + somecolumn Float64 + ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; + """ + ) + node7.query( + "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)" + ) + assert ( + node7.query( + "SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0" + ) + == "2\n" + ) + assert ( + node7.query( + "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id" + ) + == "777.777\n88.88\n99.99\n" + ) + assert ( + node7.query( + "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100" + ) + == "hello\nworld\n" + ) + + node7.query( + "INSERT INTO compression_codec_multiple_with_key SELECT toDate('2018-10-12'), number, toString(number), 1.0 FROM system.numbers LIMIT 10000" + ) + + assert ( + node7.query( + "SELECT COUNT(id) FROM compression_codec_multiple_with_key WHERE id % 10 == 0" + ) + == "1001\n" + ) + assert ( + node7.query( + "SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key" + ) + == str(777.777 + 88.88 + 99.99 + 1.0 * 10000) + "\n" + ) + assert ( + node7.query( + "SELECT count(*) FROM compression_codec_multiple_with_key GROUP BY somedate" + ) + == "10003\n" + ) diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference b/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference index cfbfadf1e676..a6afe11126ce 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference @@ -12,13 +12,24 @@ CODEC(NONE) 2018-01-01 4 4 2018-01-01 5 5 2018-01-01 6 6 +CODEC(DEFLATE_QPL) 2018-01-01 1 default_value 2018-01-01 2 default_value 2018-01-01 3 3 2018-01-01 4 4 2018-01-01 5 5 2018-01-01 6 6 -CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, NONE) +2018-01-01 7 7 +2018-01-01 8 8 +2018-01-01 1 default_value +2018-01-01 2 default_value +2018-01-01 3 3 +2018-01-01 4 4 +2018-01-01 5 5 +2018-01-01 6 6 +2018-01-01 7 7 +2018-01-01 8 8 +CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, DEFLATE_QPL, NONE) 2018-01-01 1 default_value 2018-01-01 2 default_value 2018-01-01 3 3 @@ -27,7 +38,9 @@ CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, NONE) 2018-01-01 6 6 2018-01-01 7 7 2018-01-01 8 8 -CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, NONE) -CODEC(NONE, LZ4, LZ4HC(0), ZSTD(1)) +2018-01-01 9 9 +2018-01-01 10 10 +CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, DEFLATE_QPL, NONE) +CODEC(NONE, LZ4, LZ4HC(0), ZSTD(1), DEFLATE_QPL) 2 1 diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index 85e5f8b63ad6..40a8bb4c7cbe 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -25,15 +25,23 @@ INSERT INTO alter_compression_codec VALUES('2018-01-01', 5, '5'); INSERT INTO alter_compression_codec VALUES('2018-01-01', 6, '6'); SELECT * FROM alter_compression_codec ORDER BY id; +SET enable_qpl_deflate = 1; +ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL); +SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; + +INSERT INTO alter_compression_codec VALUES('2018-01-01', 7, '7'); +INSERT INTO alter_compression_codec VALUES('2018-01-01', 8, '8'); +SELECT * FROM alter_compression_codec ORDER BY id; + OPTIMIZE TABLE alter_compression_codec FINAL; SELECT * FROM alter_compression_codec ORDER BY id; SET allow_suspicious_codecs = 1; -ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(ZSTD, LZ4HC, LZ4, LZ4, NONE); +ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(ZSTD, LZ4HC, LZ4, LZ4, DEFLATE_QPL, NONE); SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; -INSERT INTO alter_compression_codec VALUES('2018-01-01', 7, '7'); -INSERT INTO alter_compression_codec VALUES('2018-01-01', 8, '8'); +INSERT INTO alter_compression_codec VALUES('2018-01-01', 9, '9'); +INSERT INTO alter_compression_codec VALUES('2018-01-01', 10, '10'); OPTIMIZE TABLE alter_compression_codec FINAL; SELECT * FROM alter_compression_codec ORDER BY id; @@ -54,15 +62,17 @@ ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 00:00:00' CODEC(ZSTD(100)); -- { serverError 433 } +ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 00:00:00' CODEC(DEFLATE_QPL(100)); -- { serverError 378 } + DROP TABLE IF EXISTS alter_bad_codec; DROP TABLE IF EXISTS large_alter_table_00804; DROP TABLE IF EXISTS store_of_hash_00804; CREATE TABLE large_alter_table_00804 ( - somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), - data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4) + somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), DEFLATE_QPL), + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, DEFLATE_QPL), + data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4, DEFLATE_QPL) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi', min_bytes_for_wide_part = 0; INSERT INTO large_alter_table_00804 SELECT toDate('2019-01-01'), number, toString(number + rand()) FROM system.numbers LIMIT 300000; @@ -71,7 +81,7 @@ CREATE TABLE store_of_hash_00804 (hash UInt64) ENGINE = Memory(); INSERT INTO store_of_hash_00804 SELECT sum(cityHash64(*)) FROM large_alter_table_00804; -ALTER TABLE large_alter_table_00804 MODIFY COLUMN data CODEC(NONE, LZ4, LZ4HC, ZSTD); +ALTER TABLE large_alter_table_00804 MODIFY COLUMN data CODEC(NONE, LZ4, LZ4HC, ZSTD, DEFLATE_QPL); OPTIMIZE TABLE large_alter_table_00804; From d85bc02388317ed4b2743814bcc217baf1652971 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Wed, 24 May 2023 15:08:23 -0400 Subject: [PATCH 02/20] add function test for deflate_qpl --- ...4_test_custom_compression_codecs.reference | 8 ++-- .../00804_test_custom_compression_codecs.sql | 45 +++++++++++-------- ...m_compression_codes_log_storages.reference | 20 ++++----- ..._custom_compression_codes_log_storages.sql | 41 +++++++++-------- ...st_deflate_qpl_codec_compression.reference | 4 ++ ...804_test_deflate_qpl_codec_compression.sql | 32 +++++++++++++ ...804_test_delta_codec_compression.reference | 2 + .../00804_test_delta_codec_compression.sql | 38 ++++++++++++++++ 8 files changed, 140 insertions(+), 50 deletions(-) create mode 100644 tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference create mode 100644 tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference index 7bd91e5a69b9..a9cbe3d32d30 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference @@ -1,6 +1,6 @@ -1 hello 2018-12-14 1.1 aaa 5 -2 world 2018-12-15 2.2 bbb 6 -3 ! 2018-12-16 3.3 ccc 7 +1 hello 2018-12-14 1.1 aaa 5 qpl11 11 +2 world 2018-12-15 2.2 bbb 6 qpl22 22 +3 ! 2018-12-16 3.3 ccc 7 qpl33 33 2 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 @@ -9,7 +9,7 @@ 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0), DEFLATE_QPL),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), NONE, NONE, NONE, LZ4HC(0), DEFLATE_QPL),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8), DEFLATE_QPL),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8), DEFLATE_QPL)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] 7.1 xxxxxxxxxxxx [127] ['Henry'] ! diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql index c080c2fc98ee..44a0daada274 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql @@ -1,5 +1,6 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; +SET enable_qpl_deflate = 1; DROP TABLE IF EXISTS compression_codec; @@ -9,18 +10,20 @@ CREATE TABLE compression_codec( ddd Date CODEC(NONE), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), - othernum Int64 CODEC(Delta) + othernum Int64 CODEC(Delta), + qplstr String CODEC(DEFLATE_QPL), + qplnum UInt32 CODEC(DEFLATE_QPL), ) ENGINE = MergeTree() ORDER BY tuple(); -INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); -INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); -INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); +INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); +INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6,'qpl22', 22); +INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); SELECT * FROM compression_codec ORDER BY id; OPTIMIZE TABLE compression_codec FINAL; -INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); +INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); DETACH TABLE compression_codec; ATTACH TABLE compression_codec; @@ -31,25 +34,31 @@ DROP TABLE IF EXISTS compression_codec; DROP TABLE IF EXISTS bad_codec; DROP TABLE IF EXISTS params_when_no_params; +DROP TABLE IF EXISTS params_when_no_params2; DROP TABLE IF EXISTS too_many_params; DROP TABLE IF EXISTS codec_multiple_direct_specification_1; DROP TABLE IF EXISTS codec_multiple_direct_specification_2; +DROP TABLE IF EXISTS codec_multiple_direct_specification_3; DROP TABLE IF EXISTS delta_bad_params1; DROP TABLE IF EXISTS delta_bad_params2; CREATE TABLE bad_codec(id UInt64 CODEC(adssadads)) ENGINE = MergeTree() order by tuple(); -- { serverError 432 } CREATE TABLE too_many_params(id UInt64 CODEC(ZSTD(2,3,4,5))) ENGINE = MergeTree() order by tuple(); -- { serverError 431 } CREATE TABLE params_when_no_params(id UInt64 CODEC(LZ4(1))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 378 } +CREATE TABLE params_when_no_params2(id UInt64 CODEC(DEFLATE_QPL(1))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 378 } CREATE TABLE codec_multiple_direct_specification_1(id UInt64 CODEC(MULTIPLE(LZ4, ZSTD))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 432 } CREATE TABLE codec_multiple_direct_specification_2(id UInt64 CODEC(multiple(LZ4, ZSTD))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 432 } +CREATE TABLE codec_multiple_direct_specification_3(id UInt64 CODEC(multiple(LZ4, DEFLATE_QPL))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 432 } CREATE TABLE delta_bad_params1(id UInt64 CODEC(Delta(3))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 433 } CREATE TABLE delta_bad_params2(id UInt64 CODEC(Delta(16))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 433 } DROP TABLE IF EXISTS bad_codec; DROP TABLE IF EXISTS params_when_no_params; +DROP TABLE IF EXISTS params_when_no_params2; DROP TABLE IF EXISTS too_many_params; DROP TABLE IF EXISTS codec_multiple_direct_specification_1; DROP TABLE IF EXISTS codec_multiple_direct_specification_2; +DROP TABLE IF EXISTS codec_multiple_direct_specification_3; DROP TABLE IF EXISTS delta_bad_params1; DROP TABLE IF EXISTS delta_bad_params2; @@ -58,10 +67,10 @@ DROP TABLE IF EXISTS compression_codec_multiple; SET network_compression_method = 'lz4hc'; CREATE TABLE compression_codec_multiple ( - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), - data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), - ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), - somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4), DEFLATE_QPL), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8), DEFLATE_QPL), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC, DEFLATE_QPL), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD, DEFLATE_QPL) ) ENGINE = MergeTree() ORDER BY tuple(); INSERT INTO compression_codec_multiple VALUES (1, 'world', toDate('2018-10-05'), 1.1), (2, 'hello', toDate('2018-10-01'), 2.2), (3, 'buy', toDate('2018-10-11'), 3.3); @@ -85,15 +94,15 @@ SELECT sum(cityHash64(*)) FROM compression_codec_multiple; DROP TABLE IF EXISTS compression_codec_multiple_more_types; CREATE TABLE compression_codec_multiple_more_types ( - id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC), - data FixedString(12) CODEC(ZSTD, ZSTD, Delta, Delta, Delta, NONE, NONE, NONE, LZ4HC), - ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8)) + id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC, DEFLATE_QPL), + data FixedString(12) CODEC(ZSTD, ZSTD, Delta, Delta, Delta, NONE, NONE, NONE, LZ4HC, DEFLATE_QPL), + ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8), DEFLATE_QPL) ) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } CREATE TABLE compression_codec_multiple_more_types ( - id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC), - data FixedString(12) CODEC(ZSTD, ZSTD, NONE, NONE, NONE, LZ4HC), - ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8)) + id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC, DEFLATE_QPL), + data FixedString(12) CODEC(ZSTD, ZSTD, NONE, NONE, NONE, LZ4HC, DEFLATE_QPL), + ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8), DEFLATE_QPL) ) ENGINE = MergeTree() ORDER BY tuple(); SHOW CREATE TABLE compression_codec_multiple_more_types; @@ -109,9 +118,9 @@ SET network_compression_method = 'zstd'; SET network_zstd_compression_level = 5; CREATE TABLE compression_codec_multiple_with_key ( - somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), Delta, Delta), - id UInt64 CODEC(LZ4, ZSTD, Delta, NONE, LZ4HC, Delta), - data String CODEC(ZSTD(2), Delta(1), LZ4HC, NONE, LZ4, LZ4) + somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), Delta, Delta, DEFLATE_QPL), + id UInt64 CODEC(LZ4, ZSTD, Delta, NONE, LZ4HC, Delta, DEFLATE_QPL), + data String CODEC(ZSTD(2), Delta(1), LZ4HC, NONE, LZ4, LZ4, DEFLATE_QPL) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference index 8145ca998292..d64b8a77eed0 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference @@ -1,9 +1,9 @@ -CREATE TABLE default.compression_codec_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = Log -1 hello 2018-12-14 1.1 aaa 5 -2 world 2018-12-15 2.2 bbb 6 -3 ! 2018-12-16 3.3 ccc 7 +CREATE TABLE default.compression_codec_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8)),\n `qplstr` String CODEC(DEFLATE_QPL),\n `qplnum` UInt32 CODEC(DEFLATE_QPL)\n)\nENGINE = Log +1 hello 2018-12-14 1.1 aaa 5 qpl11 11 +2 world 2018-12-15 2.2 bbb 6 qpl22 22 +3 ! 2018-12-16 3.3 ccc 7 qpl33 33 2 -CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = Log +CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4), DEFLATE_QPL),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8), DEFLATE_QPL),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0), DEFLATE_QPL),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1), DEFLATE_QPL)\n)\nENGINE = Log 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 @@ -11,12 +11,12 @@ CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_tiny_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = TinyLog -1 hello 2018-12-14 1.1 aaa 5 -2 world 2018-12-15 2.2 bbb 6 -3 ! 2018-12-16 3.3 ccc 7 +CREATE TABLE default.compression_codec_tiny_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8)),\n `qplstr` String CODEC(DEFLATE_QPL),\n `qplnum` UInt32 CODEC(DEFLATE_QPL)\n)\nENGINE = TinyLog +1 hello 2018-12-14 1.1 aaa 5 qpl11 11 +2 world 2018-12-15 2.2 bbb 6 qpl22 22 +3 ! 2018-12-16 3.3 ccc 7 qpl33 33 2 -CREATE TABLE default.compression_codec_multiple_tiny_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = TinyLog +CREATE TABLE default.compression_codec_multiple_tiny_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4), DEFLATE_QPL),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8), DEFLATE_QPL),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0), DEFLATE_QPL),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1), DEFLATE_QPL)\n)\nENGINE = TinyLog 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql index fba6a2167627..113f26732e74 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql @@ -1,5 +1,6 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; +SET enable_qpl_deflate = 1; -- copy-paste for storage log @@ -11,18 +12,20 @@ CREATE TABLE compression_codec_log( ddd Date CODEC(NONE), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), - othernum Int64 CODEC(Delta) + othernum Int64 CODEC(Delta), + qplstr String CODEC(DEFLATE_QPL), + qplnum UInt32 CODEC(DEFLATE_QPL), ) ENGINE = Log(); SHOW CREATE TABLE compression_codec_log; -INSERT INTO compression_codec_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); -INSERT INTO compression_codec_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); -INSERT INTO compression_codec_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); +INSERT INTO compression_codec_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); +INSERT INTO compression_codec_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6,'qpl22', 22); +INSERT INTO compression_codec_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); SELECT * FROM compression_codec_log ORDER BY id; -INSERT INTO compression_codec_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); +INSERT INTO compression_codec_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); DETACH TABLE compression_codec_log; ATTACH TABLE compression_codec_log; @@ -34,10 +37,10 @@ DROP TABLE IF EXISTS compression_codec_log; DROP TABLE IF EXISTS compression_codec_multiple_log; CREATE TABLE compression_codec_multiple_log ( - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), - data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), - ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), - somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4), DEFLATE_QPL), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8), DEFLATE_QPL), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC, DEFLATE_QPL), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD, DEFLATE_QPL) ) ENGINE = Log(); SHOW CREATE TABLE compression_codec_multiple_log; @@ -69,18 +72,20 @@ CREATE TABLE compression_codec_tiny_log( ddd Date CODEC(NONE), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), - othernum Int64 CODEC(Delta) + othernum Int64 CODEC(Delta), + qplstr String CODEC(DEFLATE_QPL), + qplnum UInt32 CODEC(DEFLATE_QPL), ) ENGINE = TinyLog(); SHOW CREATE TABLE compression_codec_tiny_log; -INSERT INTO compression_codec_tiny_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); -INSERT INTO compression_codec_tiny_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); -INSERT INTO compression_codec_tiny_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); +INSERT INTO compression_codec_tiny_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); +INSERT INTO compression_codec_tiny_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6, 'qpl22', 22); +INSERT INTO compression_codec_tiny_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); SELECT * FROM compression_codec_tiny_log ORDER BY id; -INSERT INTO compression_codec_tiny_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); +INSERT INTO compression_codec_tiny_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); DETACH TABLE compression_codec_tiny_log; ATTACH TABLE compression_codec_tiny_log; @@ -92,10 +97,10 @@ DROP TABLE IF EXISTS compression_codec_tiny_log; DROP TABLE IF EXISTS compression_codec_multiple_tiny_log; CREATE TABLE compression_codec_multiple_tiny_log ( - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), - data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), - ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), - somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4), DEFLATE_QPL), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8), DEFLATE_QPL), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC, DEFLATE_QPL), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD, DEFLATE_QPL) ) ENGINE = TinyLog(); SHOW CREATE TABLE compression_codec_multiple_tiny_log; diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference new file mode 100644 index 000000000000..88d274d9cbac --- /dev/null +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference @@ -0,0 +1,4 @@ +1 hello 2018-12-14 1.1 aaa 5 qpl11 11 +2 world 2018-12-15 2.2 bbb 6 qpl22 22 +3 ! 2018-12-16 3.3 ccc 7 qpl33 33 +2 diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql new file mode 100644 index 000000000000..fe23e49804da --- /dev/null +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -0,0 +1,32 @@ +SET send_logs_level = 'fatal'; +SET enable_qpl_deflate = 1; + +DROP TABLE IF EXISTS compression_codec; + +CREATE TABLE compression_codec( + id UInt64 CODEC(DEFLATE_QPL), + data String CODEC(DEFLATE_QPL), + ddd Date CODEC(DEFLATE_QPL), + somenum Float64 CODEC(DEFLATE_QPL), + somestr FixedString(3) CODEC(DEFLATE_QPL), + othernum Int64 CODEC(DEFLATE_QPL), + qplstr String CODEC(DEFLATE_QPL), + qplnum UInt32 CODEC(DEFLATE_QPL), +) ENGINE = MergeTree() ORDER BY tuple(); + +INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); +INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6,'qpl22', 22); +INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); + +SELECT * FROM compression_codec ORDER BY id; + +OPTIMIZE TABLE compression_codec FINAL; + +INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); + +DETACH TABLE compression_codec; +ATTACH TABLE compression_codec; + +SELECT count(*) FROM compression_codec WHERE id = 2 GROUP BY id; + +DROP TABLE IF EXISTS compression_codec; diff --git a/tests/queries/0_stateless/00804_test_delta_codec_compression.reference b/tests/queries/0_stateless/00804_test_delta_codec_compression.reference index 949d37ed27a5..37f9d4901b37 100644 --- a/tests/queries/0_stateless/00804_test_delta_codec_compression.reference +++ b/tests/queries/0_stateless/00804_test_delta_codec_compression.reference @@ -4,3 +4,5 @@ 1 32 1 +17 +1 diff --git a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql index 25988f6474b6..f9805246662d 100644 --- a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql @@ -115,3 +115,41 @@ USING(key); DROP TABLE IF EXISTS delta_codec_string; DROP TABLE IF EXISTS default_codec_string; + +SET enable_qpl_deflate = 1; +DROP TABLE IF EXISTS delta_codec_string_qpl; +DROP TABLE IF EXISTS default_codec_string_qpl; + +CREATE TABLE delta_codec_string_qpl +( + id Float64 Codec(Delta, DEFLATE_QPL) +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; + +CREATE TABLE default_codec_string_qpl +( + id Float64 Codec(DEFLATE_QPL) +) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; + +INSERT INTO delta_codec_string_qpl SELECT concat(toString(number), toString(number % 100)) FROM numbers(1547510400, 500000); +INSERT INTO default_codec_string_qpl SELECT * from delta_codec_string_qpl; + +OPTIMIZE TABLE delta_codec_string_qpl FINAL; +OPTIMIZE TABLE default_codec_string_qpl FINAL; + +SELECT + floor(big_size / small_size) as ratio +FROM + (SELECT 1 AS key, sum(bytes_on_disk) AS small_size FROM system.parts WHERE database = currentDatabase() and table = 'delta_codec_string_qpl' and active) +INNER JOIN + (SELECT 1 AS key, sum(bytes_on_disk) as big_size FROM system.parts WHERE database = currentDatabase() and table = 'default_codec_string_qpl' and active) USING(key); + +SELECT + small_hash == big_hash +FROM + (SELECT 1 AS key, sum(cityHash64(*)) AS small_hash FROM delta_codec_string_qpl) +INNER JOIN + (SELECT 1 AS key, sum(cityHash64(*)) AS big_hash FROM default_codec_string_qpl) +USING(key); + +DROP TABLE IF EXISTS delta_codec_string_qpl; +DROP TABLE IF EXISTS default_codec_string_qpl; From 31173ab55b0926f634c2fbfc06f7d2f34410a4ff Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Wed, 24 May 2023 15:15:40 -0400 Subject: [PATCH 03/20] add sections of deflate_qpl for stress test and performance test --- tests/ci/stress.py | 1 + tests/performance/codecs_float_insert.xml | 2 ++ tests/performance/codecs_float_select.xml | 2 ++ tests/performance/codecs_int_insert.xml | 2 ++ tests/performance/codecs_int_select.xml | 2 ++ 5 files changed, 9 insertions(+) diff --git a/tests/ci/stress.py b/tests/ci/stress.py index b90448740716..b95cac9044ef 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -20,6 +20,7 @@ def get_options(i, upgrade_check): '''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i) ) client_options.append("allow_experimental_database_replicated=1") + client_options.append("enable_qpl_deflate=1") # If database name is not specified, new database is created for each functional test. # Run some threads with one database for all tests. diff --git a/tests/performance/codecs_float_insert.xml b/tests/performance/codecs_float_insert.xml index 64325d301891..25291f7f4991 100644 --- a/tests/performance/codecs_float_insert.xml +++ b/tests/performance/codecs_float_insert.xml @@ -1,6 +1,7 @@ 1 + 1 @@ -10,6 +11,7 @@ NONE LZ4 ZSTD + DEFLATE_QPL DoubleDelta Gorilla FPC diff --git a/tests/performance/codecs_float_select.xml b/tests/performance/codecs_float_select.xml index 325c140d9a0c..bb67987c75e3 100644 --- a/tests/performance/codecs_float_select.xml +++ b/tests/performance/codecs_float_select.xml @@ -1,6 +1,7 @@ 1 + 1 @@ -10,6 +11,7 @@ NONE LZ4 ZSTD + DEFLATE_QPL DoubleDelta Gorilla FPC diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index 618e20160f82..1db9ee8f7463 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -1,6 +1,7 @@ 1 + 1 @@ -10,6 +11,7 @@ NONE LZ4 ZSTD + DEFLATE_QPL Delta T64 DoubleDelta diff --git a/tests/performance/codecs_int_select.xml b/tests/performance/codecs_int_select.xml index 62c1ee16e7b7..5dc7ab487049 100644 --- a/tests/performance/codecs_int_select.xml +++ b/tests/performance/codecs_int_select.xml @@ -1,6 +1,7 @@ 1 + 1 @@ -10,6 +11,7 @@ NONE LZ4 ZSTD + DEFLATE_QPL Delta T64 DoubleDelta From cbdb408ec8330c8ce469c68e979ca208c76d0629 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Fri, 26 May 2023 12:15:34 -0400 Subject: [PATCH 04/20] add USE_QPL for buildoptions --- src/Storages/System/StorageSystemBuildOptions.cpp.in | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/System/StorageSystemBuildOptions.cpp.in b/src/Storages/System/StorageSystemBuildOptions.cpp.in index 3465e47449b9..c2a188e77506 100644 --- a/src/Storages/System/StorageSystemBuildOptions.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.cpp.in @@ -68,6 +68,7 @@ const char * auto_config_build[] "GIT_BRANCH", R"IRjaNsZIL9Yh7FQ4(@GIT_BRANCH@)IRjaNsZIL9Yh7FQ4", "GIT_DATE", "@GIT_DATE@", "GIT_COMMIT_SUBJECT", R"Gi17KJMlbGCjErEN(@GIT_COMMIT_SUBJECT@)Gi17KJMlbGCjErEN", + "USE_QPL", "@ENABLE_QPL@", nullptr, nullptr }; From f1192d59afa7ee2271d7ee6b5cb9d98bb27254a0 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Thu, 1 Jun 2023 12:42:22 -0400 Subject: [PATCH 05/20] refine patch according to comments --- .../sql-reference/statements/create/table.md | 2 +- src/Client/Connection.cpp | 2 +- src/Compression/CompressionFactory.h | 4 +- .../CompressionFactoryAdditions.cpp | 12 +++--- src/Compression/ICompressionCodec.h | 6 +-- src/Core/Settings.h | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 4 +- src/Server/TCPHandler.cpp | 2 +- src/Storages/AlterCommands.cpp | 4 +- src/Storages/Distributed/DistributedSink.cpp | 2 +- src/Storages/TTLDescription.cpp | 2 +- tests/ci/stress.py | 2 +- .../configs/enable_deflateqpl_codec.xml | 2 +- .../test_non_default_compression/test.py | 30 +++++++-------- tests/performance/codecs_float_insert.xml | 3 +- tests/performance/codecs_float_select.xml | 3 +- tests/performance/codecs_int_insert.xml | 3 +- tests/performance/codecs_int_select.xml | 3 +- ...04_test_alter_compression_codecs.reference | 4 +- .../00804_test_alter_compression_codecs.sql | 10 ++--- ...4_test_custom_compression_codecs.reference | 6 +-- .../00804_test_custom_compression_codecs.sql | 13 +++---- ..._custom_compression_codes_log_storages.sql | 2 +- ...st_deflate_qpl_codec_compression.reference | 6 +-- ...804_test_deflate_qpl_codec_compression.sql | 16 ++++---- ...804_test_delta_codec_compression.reference | 2 - .../00804_test_delta_codec_compression.sql | 38 ------------------- 27 files changed, 70 insertions(+), 115 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index b0865ad28964..d0e17410791c 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -380,7 +380,7 @@ High compression levels are useful for asymmetric scenarios, like compress once, `DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply: -- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_qpl_deflate=1`. +- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_qpl_deflate_codec=1`. - DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details. - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details. - DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled. diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 68bc3b39a560..ac8e6654e849 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -588,7 +588,7 @@ void Connection::sendQuery( if (method == "ZSTD") level = settings->network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_qpl_deflate); + CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_qpl_deflate_codec); compression_codec = CompressionCodecFactory::instance().get(method, level); } else diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index 1fdaf4f1c713..e020e51bb09b 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -40,10 +40,10 @@ class CompressionCodecFactory final : private boost::noncopyable CompressionCodecPtr getDefaultCodec() const; /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const; + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const; /// Validate codecs AST specified by user - void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const; + void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const; /// Get codec by AST and possible column_type. Some codecs can use /// information about type to improve inner settings, but every codec should diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index 2630326238af..b4a2d96cf39e 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -34,7 +34,7 @@ namespace ErrorCodes void CompressionCodecFactory::validateCodec( - const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const + const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const { if (family_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty"); @@ -43,13 +43,13 @@ void CompressionCodecFactory::validateCodec( { auto literal = std::make_shared(static_cast(*level)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), - {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate); + {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec); } else { auto identifier = std::make_shared(Poco::toUpper(family_name)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), - {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate); + {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec); } } @@ -77,7 +77,7 @@ bool innerDataTypeIsFloat(const DataTypePtr & type) } ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( - const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate) const + const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const { if (const auto * func = ast->as()) { @@ -159,10 +159,10 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( " You can enable it with the 'allow_experimental_codecs' setting.", codec_family_name); - if (!enable_qpl_deflate && result_codec->isDeflateQplCompression()) + if (!enable_qpl_deflate_codec && result_codec->isDeflateQplCompression()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec {} is disabled by default." - " You can enable it with the 'enable_qpl_deflate' setting.", + " You can enable it with the 'enable_qpl_deflate_codec' setting.", codec_family_name); codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index d92ad3fc718f..f7e8f4e43d23 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -109,12 +109,12 @@ class ICompressionCodec : private boost::noncopyable /// It will not be allowed to use unless the user will turn off the safety switch. virtual bool isExperimental() const { return false; } - /// If it does nothing. - virtual bool isNone() const { return false; } - /// This is a knob for Deflate QPL codec. virtual bool isDeflateQplCompression() const { return false; } + /// If it does nothing. + virtual bool isNone() const { return false; } + protected: /// This is used for fuzz testing friend int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index c6a2069e6ae2..4aae8f5d5728 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -319,7 +319,7 @@ class IColumn; M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ - M(Bool, enable_qpl_deflate, false, "If it is set to true, allow to use deflate_qpl for compression.", 0) \ + M(Bool, enable_qpl_deflate_codec, false, "If it is set to true, allow usage of the DEFLATE_QPL codec.", 0) \ M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 5c22b46b3607..ddb53bbbfaa4 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -571,7 +571,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; - bool enable_qpl_deflate = attach || context_->getSettingsRef().enable_qpl_deflate; + bool enable_qpl_deflate_codec = attach || context_->getSettingsRef().enable_qpl_deflate_codec; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); @@ -632,7 +632,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.default_specifier == "ALIAS") throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_qpl_deflate); + col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_qpl_deflate_codec); } if (col_decl.ttl) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 96c585e7d164..b43fef9dd541 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1775,7 +1775,7 @@ void TCPHandler::initBlockOutput(const Block & block) if (state.compression == Protocol::Compression::Enable) { - CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_qpl_deflate); + CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_qpl_deflate_codec); state.maybe_compressed_out = std::make_shared( *out, CompressionCodecFactory::instance().get(method, level)); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index ecbddfc3e2a0..73d7be8dc569 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1067,7 +1067,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const "this column name is reserved for lightweight delete feature", backQuote(column_name)); if (command.codec) - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); all_columns.add(ColumnDescription(column_name, command.data_type)); } @@ -1093,7 +1093,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const { if (all_columns.hasAlias(column_name)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); } auto column_default = all_columns.getDefault(column_name); if (column_default) diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index ce1dbde8eaec..e383890d1f7a 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -733,7 +733,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const if (compression_method == "ZSTD") compression_level = settings.network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_qpl_deflate); + CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_qpl_deflate_codec); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index f5209cbdff6a..a437465b3fe1 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -285,7 +285,7 @@ TTLDescription TTLDescription::getTTLFromAST( { result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate); + ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); } } diff --git a/tests/ci/stress.py b/tests/ci/stress.py index b95cac9044ef..e5ceb251d0f3 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -20,7 +20,7 @@ def get_options(i, upgrade_check): '''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i) ) client_options.append("allow_experimental_database_replicated=1") - client_options.append("enable_qpl_deflate=1") + client_options.append("enable_qpl_deflate_codec=1") # If database name is not specified, new database is created for each functional test. # Run some threads with one database for all tests. diff --git a/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml b/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml index 46e9e43ca276..521b0fd663cb 100644 --- a/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml +++ b/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml @@ -1,7 +1,7 @@ - 1 + 1 diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index e69b32daae08..e1a9c1ae5409 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -38,17 +38,17 @@ ) node6 = cluster.add_instance( "node6", - main_configs=["configs/allow_experimental_codecs.xml"], - user_configs=["configs/allow_suspicious_codecs.xml"], -) -node7 = cluster.add_instance( - "node7", main_configs=["configs/deflateqpl_compression_by_default.xml"], user_configs=[ - "configs/enable_deflateqpl_codec.xml", "configs/allow_suspicious_codecs.xml", + "configs/enable_deflateqpl_codec.xml", ], ) +node7 = cluster.add_instance( + "node7", + main_configs=["configs/allow_experimental_codecs.xml"], + user_configs=["configs/allow_suspicious_codecs.xml"], +) @pytest.fixture(scope="module") def start_cluster(): @@ -253,7 +253,7 @@ def test_uncompressed_cache_plus_zstd_codec(start_cluster): ) def test_preconfigured_deflateqpl_codec(start_cluster): - node7.query( + node6.query( """ CREATE TABLE compression_codec_multiple_with_key ( somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), DEFLATE_QPL), @@ -263,46 +263,46 @@ def test_preconfigured_deflateqpl_codec(start_cluster): ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2; """ ) - node7.query( + node6.query( "INSERT INTO compression_codec_multiple_with_key VALUES(toDate('2018-10-12'), 100000, 'hello', 88.88), (toDate('2018-10-12'), 100002, 'world', 99.99), (toDate('2018-10-12'), 1111, '!', 777.777)" ) assert ( - node7.query( + node6.query( "SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0" ) == "2\n" ) assert ( - node7.query( + node6.query( "SELECT DISTINCT somecolumn FROM compression_codec_multiple_with_key ORDER BY id" ) == "777.777\n88.88\n99.99\n" ) assert ( - node7.query( + node6.query( "SELECT data FROM compression_codec_multiple_with_key WHERE id >= 1112 AND somedate = toDate('2018-10-12') AND somecolumn <= 100" ) == "hello\nworld\n" ) - node7.query( + node6.query( "INSERT INTO compression_codec_multiple_with_key SELECT toDate('2018-10-12'), number, toString(number), 1.0 FROM system.numbers LIMIT 10000" ) assert ( - node7.query( + node6.query( "SELECT COUNT(id) FROM compression_codec_multiple_with_key WHERE id % 10 == 0" ) == "1001\n" ) assert ( - node7.query( + node6.query( "SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key" ) == str(777.777 + 88.88 + 99.99 + 1.0 * 10000) + "\n" ) assert ( - node7.query( + node6.query( "SELECT count(*) FROM compression_codec_multiple_with_key GROUP BY somedate" ) == "10003\n" diff --git a/tests/performance/codecs_float_insert.xml b/tests/performance/codecs_float_insert.xml index 25291f7f4991..be0935ad4cf0 100644 --- a/tests/performance/codecs_float_insert.xml +++ b/tests/performance/codecs_float_insert.xml @@ -1,7 +1,7 @@ 1 - 1 + 1 @@ -11,7 +11,6 @@ NONE LZ4 ZSTD - DEFLATE_QPL DoubleDelta Gorilla FPC diff --git a/tests/performance/codecs_float_select.xml b/tests/performance/codecs_float_select.xml index bb67987c75e3..844ab4508d89 100644 --- a/tests/performance/codecs_float_select.xml +++ b/tests/performance/codecs_float_select.xml @@ -1,7 +1,7 @@ 1 - 1 + 1 @@ -11,7 +11,6 @@ NONE LZ4 ZSTD - DEFLATE_QPL DoubleDelta Gorilla FPC diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index 1db9ee8f7463..d5f128101189 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -1,7 +1,7 @@ 1 - 1 + 1 @@ -11,7 +11,6 @@ NONE LZ4 ZSTD - DEFLATE_QPL Delta T64 DoubleDelta diff --git a/tests/performance/codecs_int_select.xml b/tests/performance/codecs_int_select.xml index 5dc7ab487049..06b2c2a73f3f 100644 --- a/tests/performance/codecs_int_select.xml +++ b/tests/performance/codecs_int_select.xml @@ -1,7 +1,7 @@ 1 - 1 + 1 @@ -11,7 +11,6 @@ NONE LZ4 ZSTD - DEFLATE_QPL Delta T64 DoubleDelta diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference b/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference index a6afe11126ce..5c77a102740d 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference @@ -12,15 +12,13 @@ CODEC(NONE) 2018-01-01 4 4 2018-01-01 5 5 2018-01-01 6 6 -CODEC(DEFLATE_QPL) 2018-01-01 1 default_value 2018-01-01 2 default_value 2018-01-01 3 3 2018-01-01 4 4 2018-01-01 5 5 2018-01-01 6 6 -2018-01-01 7 7 -2018-01-01 8 8 +CODEC(DEFLATE_QPL) 2018-01-01 1 default_value 2018-01-01 2 default_value 2018-01-01 3 3 diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index 40a8bb4c7cbe..5b8b73270a2e 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -25,7 +25,10 @@ INSERT INTO alter_compression_codec VALUES('2018-01-01', 5, '5'); INSERT INTO alter_compression_codec VALUES('2018-01-01', 6, '6'); SELECT * FROM alter_compression_codec ORDER BY id; -SET enable_qpl_deflate = 1; +OPTIMIZE TABLE alter_compression_codec FINAL; +SELECT * FROM alter_compression_codec ORDER BY id; + +SET enable_qpl_deflate_codec = 1; ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL); SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; @@ -33,9 +36,6 @@ INSERT INTO alter_compression_codec VALUES('2018-01-01', 7, '7'); INSERT INTO alter_compression_codec VALUES('2018-01-01', 8, '8'); SELECT * FROM alter_compression_codec ORDER BY id; -OPTIMIZE TABLE alter_compression_codec FINAL; -SELECT * FROM alter_compression_codec ORDER BY id; - SET allow_suspicious_codecs = 1; ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(ZSTD, LZ4HC, LZ4, LZ4, DEFLATE_QPL, NONE); SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; @@ -62,7 +62,7 @@ ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 00:00:00' CODEC(ZSTD(100)); -- { serverError 433 } -ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 00:00:00' CODEC(DEFLATE_QPL(100)); -- { serverError 378 } +ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 00:00:00' CODEC(DEFLATE_QPL(100)); -- { serverError DATA_TYPE_CANNOT_HAVE_ARGUMENTS } DROP TABLE IF EXISTS alter_bad_codec; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference index a9cbe3d32d30..8b51d65004a3 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference @@ -1,6 +1,6 @@ -1 hello 2018-12-14 1.1 aaa 5 qpl11 11 -2 world 2018-12-15 2.2 bbb 6 qpl22 22 -3 ! 2018-12-16 3.3 ccc 7 qpl33 33 +1 hello 2018-12-14 2018-12-14 1.1 aaa 5 +2 world 2018-12-15 2018-12-15 2.2 bbb 6 +3 ! 2018-12-16 2018-12-16 3.3 ccc 7 2 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql index 44a0daada274..47ec268bfec1 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql @@ -1,6 +1,6 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; -SET enable_qpl_deflate = 1; +SET enable_qpl_deflate_codec = 1; DROP TABLE IF EXISTS compression_codec; @@ -8,22 +8,21 @@ CREATE TABLE compression_codec( id UInt64 CODEC(LZ4), data String CODEC(ZSTD), ddd Date CODEC(NONE), + ddd32 Date32 CODEC(DEFLATE_QPL), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), othernum Int64 CODEC(Delta), - qplstr String CODEC(DEFLATE_QPL), - qplnum UInt32 CODEC(DEFLATE_QPL), ) ENGINE = MergeTree() ORDER BY tuple(); -INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); -INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6,'qpl22', 22); -INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); +INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), toDate32('2018-12-14'), 1.1, 'aaa', 5); +INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), toDate32('2018-12-15'), 2.2, 'bbb', 6); +INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), toDate32('2018-12-16'), 3.3, 'ccc', 7); SELECT * FROM compression_codec ORDER BY id; OPTIMIZE TABLE compression_codec FINAL; -INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); +INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), toDate32('2018-12-13'), 4.4, 'ddd', 8); DETACH TABLE compression_codec; ATTACH TABLE compression_codec; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql index 113f26732e74..bcd092778242 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql @@ -1,6 +1,6 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; -SET enable_qpl_deflate = 1; +SET enable_qpl_deflate_codec = 1; -- copy-paste for storage log diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference index 88d274d9cbac..276747f82336 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference @@ -1,4 +1,4 @@ -1 hello 2018-12-14 1.1 aaa 5 qpl11 11 -2 world 2018-12-15 2.2 bbb 6 qpl22 22 -3 ! 2018-12-16 3.3 ccc 7 qpl33 33 +1 hello 2018-12-14 2018-12-14 1.1 aaa 5 [1,2,3] {'k1':1,'k2':2} (1,2) +2 world 2018-12-15 2018-12-15 2.2 bbb 6 [4,5,6] {'k3':3,'k4':4} (3,4) +3 ! 2018-12-16 2018-12-16 3.3 ccc 7 [7,8,9] {'k5':5,'k6':6} (5,6) 2 diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql index fe23e49804da..64e66d475227 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -1,5 +1,5 @@ SET send_logs_level = 'fatal'; -SET enable_qpl_deflate = 1; +SET enable_qpl_deflate_codec = 1; DROP TABLE IF EXISTS compression_codec; @@ -7,22 +7,24 @@ CREATE TABLE compression_codec( id UInt64 CODEC(DEFLATE_QPL), data String CODEC(DEFLATE_QPL), ddd Date CODEC(DEFLATE_QPL), + ddd32 Date32 CODEC(DEFLATE_QPL), somenum Float64 CODEC(DEFLATE_QPL), somestr FixedString(3) CODEC(DEFLATE_QPL), othernum Int64 CODEC(DEFLATE_QPL), - qplstr String CODEC(DEFLATE_QPL), - qplnum UInt32 CODEC(DEFLATE_QPL), + somearray Array(UInt8) CODEC(DEFLATE_QPL), + somemap Map(String, UInt32) CODEC(DEFLATE_QPL), + sometuple Tuple(UInt16, UInt64) CODEC(DEFLATE_QPL), ) ENGINE = MergeTree() ORDER BY tuple(); -INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); -INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6,'qpl22', 22); -INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); +INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), toDate32('2018-12-14'), 1.1, 'aaa', 5, [1,2,3], map('k1',1,'k2',2), tuple(1,2)); +INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), toDate32('2018-12-15'), 2.2, 'bbb', 6, [4,5,6], map('k3',3,'k4',4), tuple(3,4)); +INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), toDate32('2018-12-16'), 3.3, 'ccc', 7, [7,8,9], map('k5',5,'k6',6), tuple(5,6)); SELECT * FROM compression_codec ORDER BY id; OPTIMIZE TABLE compression_codec FINAL; -INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); +INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), toDate32('2018-12-13'), 4.4, 'ddd', 8, [10,11,12], map('k7',7,'k8',8), tuple(7,8)); DETACH TABLE compression_codec; ATTACH TABLE compression_codec; diff --git a/tests/queries/0_stateless/00804_test_delta_codec_compression.reference b/tests/queries/0_stateless/00804_test_delta_codec_compression.reference index 37f9d4901b37..949d37ed27a5 100644 --- a/tests/queries/0_stateless/00804_test_delta_codec_compression.reference +++ b/tests/queries/0_stateless/00804_test_delta_codec_compression.reference @@ -4,5 +4,3 @@ 1 32 1 -17 -1 diff --git a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql index f9805246662d..25988f6474b6 100644 --- a/tests/queries/0_stateless/00804_test_delta_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_delta_codec_compression.sql @@ -115,41 +115,3 @@ USING(key); DROP TABLE IF EXISTS delta_codec_string; DROP TABLE IF EXISTS default_codec_string; - -SET enable_qpl_deflate = 1; -DROP TABLE IF EXISTS delta_codec_string_qpl; -DROP TABLE IF EXISTS default_codec_string_qpl; - -CREATE TABLE delta_codec_string_qpl -( - id Float64 Codec(Delta, DEFLATE_QPL) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; - -CREATE TABLE default_codec_string_qpl -( - id Float64 Codec(DEFLATE_QPL) -) ENGINE MergeTree() ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, compress_marks = false, compress_primary_key=false; - -INSERT INTO delta_codec_string_qpl SELECT concat(toString(number), toString(number % 100)) FROM numbers(1547510400, 500000); -INSERT INTO default_codec_string_qpl SELECT * from delta_codec_string_qpl; - -OPTIMIZE TABLE delta_codec_string_qpl FINAL; -OPTIMIZE TABLE default_codec_string_qpl FINAL; - -SELECT - floor(big_size / small_size) as ratio -FROM - (SELECT 1 AS key, sum(bytes_on_disk) AS small_size FROM system.parts WHERE database = currentDatabase() and table = 'delta_codec_string_qpl' and active) -INNER JOIN - (SELECT 1 AS key, sum(bytes_on_disk) as big_size FROM system.parts WHERE database = currentDatabase() and table = 'default_codec_string_qpl' and active) USING(key); - -SELECT - small_hash == big_hash -FROM - (SELECT 1 AS key, sum(cityHash64(*)) AS small_hash FROM delta_codec_string_qpl) -INNER JOIN - (SELECT 1 AS key, sum(cityHash64(*)) AS big_hash FROM default_codec_string_qpl) -USING(key); - -DROP TABLE IF EXISTS delta_codec_string_qpl; -DROP TABLE IF EXISTS default_codec_string_qpl; From aae281eb7df6ce8e00d872d3ef0d0558781a5f1a Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 1 Jun 2023 15:49:52 +0200 Subject: [PATCH 06/20] Update codecs_float_insert.xml --- tests/performance/codecs_float_insert.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/codecs_float_insert.xml b/tests/performance/codecs_float_insert.xml index be0935ad4cf0..64325d301891 100644 --- a/tests/performance/codecs_float_insert.xml +++ b/tests/performance/codecs_float_insert.xml @@ -1,7 +1,6 @@ 1 - 1 From dc93b6324ee505228b96791db629b7437f6db931 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 1 Jun 2023 15:50:28 +0200 Subject: [PATCH 07/20] Update codecs_float_select.xml --- tests/performance/codecs_float_select.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/codecs_float_select.xml b/tests/performance/codecs_float_select.xml index 844ab4508d89..325c140d9a0c 100644 --- a/tests/performance/codecs_float_select.xml +++ b/tests/performance/codecs_float_select.xml @@ -1,7 +1,6 @@ 1 - 1 From 7043db669e4e445764d99cd749cfef99d3f437cf Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 1 Jun 2023 15:50:40 +0200 Subject: [PATCH 08/20] Update codecs_int_insert.xml --- tests/performance/codecs_int_insert.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index d5f128101189..618e20160f82 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -1,7 +1,6 @@ 1 - 1 From 4d7364af97893c4457a86a064628ff478d900c05 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 1 Jun 2023 15:50:49 +0200 Subject: [PATCH 09/20] Update codecs_int_select.xml --- tests/performance/codecs_int_select.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/performance/codecs_int_select.xml b/tests/performance/codecs_int_select.xml index 06b2c2a73f3f..62c1ee16e7b7 100644 --- a/tests/performance/codecs_int_select.xml +++ b/tests/performance/codecs_int_select.xml @@ -1,7 +1,6 @@ 1 - 1 From 1f928f2d3d0eea55ff1743cea386162fd87fed92 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 1 Jun 2023 15:53:48 +0200 Subject: [PATCH 10/20] Update StorageSystemBuildOptions.cpp.in --- src/Storages/System/StorageSystemBuildOptions.cpp.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/System/StorageSystemBuildOptions.cpp.in b/src/Storages/System/StorageSystemBuildOptions.cpp.in index c2a188e77506..c2d35c96ce5c 100644 --- a/src/Storages/System/StorageSystemBuildOptions.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.cpp.in @@ -64,11 +64,11 @@ const char * auto_config_build[] "USE_ARROW", "@USE_ARROW@", "USE_ORC", "@USE_ORC@", "USE_MSGPACK", "@USE_MSGPACK@", + "USE_QPL", "@ENABLE_QPL@", "GIT_HASH", "@GIT_HASH@", "GIT_BRANCH", R"IRjaNsZIL9Yh7FQ4(@GIT_BRANCH@)IRjaNsZIL9Yh7FQ4", "GIT_DATE", "@GIT_DATE@", "GIT_COMMIT_SUBJECT", R"Gi17KJMlbGCjErEN(@GIT_COMMIT_SUBJECT@)Gi17KJMlbGCjErEN", - "USE_QPL", "@ENABLE_QPL@", nullptr, nullptr }; From 1aa158909e434438733504d2dbcd9ea9d113e41b Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 9 Jun 2023 12:38:38 +0000 Subject: [PATCH 11/20] enable_qpl_deflate_codec --> enable_deflate_qpl_codec --- docs/en/sql-reference/statements/create/table.md | 2 +- src/Client/Connection.cpp | 2 +- src/Compression/CompressionFactory.h | 4 ++-- src/Compression/CompressionFactoryAdditions.cpp | 12 ++++++------ src/Core/Settings.h | 2 +- src/Interpreters/InterpreterCreateQuery.cpp | 4 ++-- src/Server/TCPHandler.cpp | 2 +- src/Storages/AlterCommands.cpp | 4 ++-- src/Storages/Distributed/DistributedSink.cpp | 2 +- src/Storages/TTLDescription.cpp | 2 +- tests/ci/stress.py | 2 +- .../configs/enable_deflateqpl_codec.xml | 2 +- .../00804_test_alter_compression_codecs.sql | 2 +- .../00804_test_custom_compression_codecs.sql | 2 +- ...04_test_custom_compression_codes_log_storages.sql | 2 +- .../00804_test_deflate_qpl_codec_compression.sql | 2 +- 16 files changed, 24 insertions(+), 24 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index d0e17410791c..496ecdbda7ba 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -380,7 +380,7 @@ High compression levels are useful for asymmetric scenarios, like compress once, `DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply: -- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_qpl_deflate_codec=1`. +- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_deflate_qpl_codec = 1`. - DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details. - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details. - DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled. diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index ac8e6654e849..636532ade4b6 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -588,7 +588,7 @@ void Connection::sendQuery( if (method == "ZSTD") level = settings->network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_qpl_deflate_codec); + CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_deflate_qpl_codec); compression_codec = CompressionCodecFactory::instance().get(method, level); } else diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index e020e51bb09b..4f2627587a39 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -40,10 +40,10 @@ class CompressionCodecFactory final : private boost::noncopyable CompressionCodecPtr getDefaultCodec() const; /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) - ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const; + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const; /// Validate codecs AST specified by user - void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const; + void validateCodec(const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const; /// Get codec by AST and possible column_type. Some codecs can use /// information about type to improve inner settings, but every codec should diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index b4a2d96cf39e..46f7e2653c21 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -34,7 +34,7 @@ namespace ErrorCodes void CompressionCodecFactory::validateCodec( - const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const + const String & family_name, std::optional level, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const { if (family_name.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty"); @@ -43,13 +43,13 @@ void CompressionCodecFactory::validateCodec( { auto literal = std::make_shared(static_cast(*level)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), - {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec); + {}, sanity_check, allow_experimental_codecs, enable_deflate_qpl_codec); } else { auto identifier = std::make_shared(Poco::toUpper(family_name)); validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), - {}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec); + {}, sanity_check, allow_experimental_codecs, enable_deflate_qpl_codec); } } @@ -77,7 +77,7 @@ bool innerDataTypeIsFloat(const DataTypePtr & type) } ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( - const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const + const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const { if (const auto * func = ast->as()) { @@ -159,10 +159,10 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( " You can enable it with the 'allow_experimental_codecs' setting.", codec_family_name); - if (!enable_qpl_deflate_codec && result_codec->isDeflateQplCompression()) + if (!enable_deflate_qpl_codec && result_codec->isDeflateQplCompression()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec {} is disabled by default." - " You can enable it with the 'enable_qpl_deflate_codec' setting.", + " You can enable it with the 'enable_deflate_qpl_codec' setting.", codec_family_name); codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 4aae8f5d5728..e0034174597d 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -319,7 +319,7 @@ class IColumn; M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ - M(Bool, enable_qpl_deflate_codec, false, "If it is set to true, allow usage of the DEFLATE_QPL codec.", 0) \ + M(Bool, enable_deflate_qpl_codec, false, "Enable/disable the DEFLATE_QPL codec.", 0) \ M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index ddb53bbbfaa4..d0bb3dd389f1 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -571,7 +571,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; - bool enable_qpl_deflate_codec = attach || context_->getSettingsRef().enable_qpl_deflate_codec; + bool enable_deflate_qpl_codec = attach || context_->getSettingsRef().enable_deflate_qpl_codec; ColumnsDescription res; auto name_type_it = column_names_and_types.begin(); @@ -632,7 +632,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription( if (col_decl.default_specifier == "ALIAS") throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_qpl_deflate_codec); + col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_deflate_qpl_codec); } if (col_decl.ttl) diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index b43fef9dd541..50e9d50e2f60 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -1775,7 +1775,7 @@ void TCPHandler::initBlockOutput(const Block & block) if (state.compression == Protocol::Compression::Enable) { - CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_qpl_deflate_codec); + CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_deflate_qpl_codec); state.maybe_compressed_out = std::make_shared( *out, CompressionCodecFactory::instance().get(method, level)); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 73d7be8dc569..a9247f9b898b 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -1067,7 +1067,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const "this column name is reserved for lightweight delete feature", backQuote(column_name)); if (command.codec) - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec); all_columns.add(ColumnDescription(column_name, command.data_type)); } @@ -1093,7 +1093,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const { if (all_columns.hasAlias(column_name)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); - CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); + CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec); } auto column_default = all_columns.getDefault(column_name); if (column_default) diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index e383890d1f7a..1e1c911920eb 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -733,7 +733,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const if (compression_method == "ZSTD") compression_level = settings.network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_qpl_deflate_codec); + CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enale_deflate_qpl_codec); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index a437465b3fe1..f601fed06aca 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -285,7 +285,7 @@ TTLDescription TTLDescription::getTTLFromAST( { result.recompression_codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( - ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); + ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec); } } diff --git a/tests/ci/stress.py b/tests/ci/stress.py index e5ceb251d0f3..6d17384c63fe 100755 --- a/tests/ci/stress.py +++ b/tests/ci/stress.py @@ -20,7 +20,7 @@ def get_options(i, upgrade_check): '''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i) ) client_options.append("allow_experimental_database_replicated=1") - client_options.append("enable_qpl_deflate_codec=1") + client_options.append("enable_deflate_qpl_codec=1") # If database name is not specified, new database is created for each functional test. # Run some threads with one database for all tests. diff --git a/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml b/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml index 521b0fd663cb..24e101e0e3fa 100644 --- a/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml +++ b/tests/integration/test_non_default_compression/configs/enable_deflateqpl_codec.xml @@ -1,7 +1,7 @@ - 1 + 1 diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index 5b8b73270a2e..fd9855e82d33 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -28,7 +28,7 @@ SELECT * FROM alter_compression_codec ORDER BY id; OPTIMIZE TABLE alter_compression_codec FINAL; SELECT * FROM alter_compression_codec ORDER BY id; -SET enable_qpl_deflate_codec = 1; +SET enable_deflate_qpl_codec = 1; ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL); SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql index 47ec268bfec1..89e77f758a7a 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql @@ -1,6 +1,6 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; -SET enable_qpl_deflate_codec = 1; +SET enable_deflate_qpl_codec = 1; DROP TABLE IF EXISTS compression_codec; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql index bcd092778242..a629df2666d2 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql @@ -1,6 +1,6 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; -SET enable_qpl_deflate_codec = 1; +SET enable_deflate_qpl_codec = 1; -- copy-paste for storage log diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql index 64e66d475227..5a56fc0d5762 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -1,5 +1,5 @@ SET send_logs_level = 'fatal'; -SET enable_qpl_deflate_codec = 1; +SET enable_deflate_qpl_codec = 1; DROP TABLE IF EXISTS compression_codec; From c538506f2e3ba0716dcc2f13f63bb4edc1f6f33e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 9 Jun 2023 20:50:17 +0000 Subject: [PATCH 12/20] More fixes --- docs/en/sql-reference/statements/create/table.md | 4 ++++ src/Compression/CompressionCodecDeflateQpl.h | 3 +-- src/Compression/CompressionFactoryAdditions.cpp | 2 +- src/Compression/ICompressionCodec.h | 4 ++-- src/Storages/Distributed/DistributedSink.cpp | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index 496ecdbda7ba..1a72f89fb1f3 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -385,6 +385,10 @@ High compression levels are useful for asymmetric scenarios, like compress once, - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details. - DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled. +:::note +DEFLATE_QPL is not available in ClickHouse Cloud. +::: + ### Specialized Codecs These codecs are designed to make compression more effective by using specific features of data. Some of these codecs do not compress data themself. Instead, they prepare the data for a common purpose codec, which compresses it better than without this preparation. diff --git a/src/Compression/CompressionCodecDeflateQpl.h b/src/Compression/CompressionCodecDeflateQpl.h index 13aa8733b54a..8d73568707e6 100644 --- a/src/Compression/CompressionCodecDeflateQpl.h +++ b/src/Compression/CompressionCodecDeflateQpl.h @@ -98,8 +98,7 @@ class CompressionCodecDeflateQpl final : public ICompressionCodec protected: bool isCompression() const override { return true; } bool isGenericCompression() const override { return true; } - bool isExperimental() const override { return false; } - bool isDeflateQplCompression() const override { return true; } + bool isDeflateQpl() const override { return true; } UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override; void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override; diff --git a/src/Compression/CompressionFactoryAdditions.cpp b/src/Compression/CompressionFactoryAdditions.cpp index 46f7e2653c21..98e9e7480da1 100644 --- a/src/Compression/CompressionFactoryAdditions.cpp +++ b/src/Compression/CompressionFactoryAdditions.cpp @@ -159,7 +159,7 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( " You can enable it with the 'allow_experimental_codecs' setting.", codec_family_name); - if (!enable_deflate_qpl_codec && result_codec->isDeflateQplCompression()) + if (!enable_deflate_qpl_codec && result_codec->isDeflateQpl()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec {} is disabled by default." " You can enable it with the 'enable_deflate_qpl_codec' setting.", diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index f7e8f4e43d23..6630838fa644 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -109,8 +109,8 @@ class ICompressionCodec : private boost::noncopyable /// It will not be allowed to use unless the user will turn off the safety switch. virtual bool isExperimental() const { return false; } - /// This is a knob for Deflate QPL codec. - virtual bool isDeflateQplCompression() const { return false; } + /// Is this the DEFLATE_QPL codec? + virtual bool isDeflateQpl() const { return false; } /// If it does nothing. virtual bool isNone() const { return false; } diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index 1e1c911920eb..875764f76331 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -733,7 +733,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const if (compression_method == "ZSTD") compression_level = settings.network_zstd_compression_level; - CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enale_deflate_qpl_codec); + CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_deflate_qpl_codec); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); /// tmp directory is used to ensure atomicity of transactions From ff96c4c0d8898c15e1aea876267c65ec8b0c69f0 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 10 Jun 2023 12:09:47 +0000 Subject: [PATCH 13/20] Fix black --- tests/integration/test_non_default_compression/test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_non_default_compression/test.py b/tests/integration/test_non_default_compression/test.py index e1a9c1ae5409..18e2eb43813a 100644 --- a/tests/integration/test_non_default_compression/test.py +++ b/tests/integration/test_non_default_compression/test.py @@ -50,6 +50,7 @@ user_configs=["configs/allow_suspicious_codecs.xml"], ) + @pytest.fixture(scope="module") def start_cluster(): try: @@ -252,6 +253,7 @@ def test_uncompressed_cache_plus_zstd_codec(start_cluster): == "10000\n" ) + def test_preconfigured_deflateqpl_codec(start_cluster): node6.query( """ @@ -268,7 +270,7 @@ def test_preconfigured_deflateqpl_codec(start_cluster): ) assert ( node6.query( - "SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0" + "SELECT COUNT(*) FROM compression_codec_multiple_with_key WHERE id % 2 == 0" ) == "2\n" ) @@ -296,9 +298,7 @@ def test_preconfigured_deflateqpl_codec(start_cluster): == "1001\n" ) assert ( - node6.query( - "SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key" - ) + node6.query("SELECT SUM(somecolumn) FROM compression_codec_multiple_with_key") == str(777.777 + 88.88 + 99.99 + 1.0 * 10000) + "\n" ) assert ( From ffb941624bc971886212e0745716e79688a154a1 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sat, 10 Jun 2023 17:01:44 +0000 Subject: [PATCH 14/20] Exclude some tests with QPL from fasttest --- .../0_stateless/00804_test_alter_compression_codecs.sql | 3 +++ .../0_stateless/00804_test_custom_compression_codecs.sql | 3 +++ .../00804_test_custom_compression_codes_log_storages.sql | 3 +++ .../0_stateless/00804_test_deflate_qpl_codec_compression.sql | 3 +++ 4 files changed, 12 insertions(+) diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index fd9855e82d33..eb1abda9a21a 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -1,3 +1,6 @@ +--Tags: no-fasttest +-- no-fasttest because DEFLATE_QPL isn't available in fasttest + SET send_logs_level = 'fatal'; DROP TABLE IF EXISTS alter_compression_codec; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql index 89e77f758a7a..df74620a2011 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql @@ -1,3 +1,6 @@ +--Tags: no-fasttest +-- no-fasttest because DEFLATE_QPL isn't available in fasttest + SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; SET enable_deflate_qpl_codec = 1; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql index a629df2666d2..67c0074c58fa 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql @@ -1,3 +1,6 @@ +--Tags: no-fasttest +-- no-fasttest because DEFLATE_QPL isn't available in fasttest + SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; SET enable_deflate_qpl_codec = 1; diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql index 5a56fc0d5762..a46272112a9c 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -1,3 +1,6 @@ +--Tags: no-fasttest +-- no-fasttest because DEFLATE_QPL isn't available in fasttest + SET send_logs_level = 'fatal'; SET enable_deflate_qpl_codec = 1; From 48e03ac92a457d612dd8b2e4838dce1e47e51109 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 11 Jun 2023 14:33:21 +0000 Subject: [PATCH 15/20] Revert "Exclude some tests with QPL from fasttest" This reverts commit ffb941624bc971886212e0745716e79688a154a1. --- .../0_stateless/00804_test_alter_compression_codecs.sql | 3 --- .../0_stateless/00804_test_custom_compression_codecs.sql | 3 --- .../00804_test_custom_compression_codes_log_storages.sql | 3 --- .../0_stateless/00804_test_deflate_qpl_codec_compression.sql | 3 --- 4 files changed, 12 deletions(-) diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index eb1abda9a21a..fd9855e82d33 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -1,6 +1,3 @@ ---Tags: no-fasttest --- no-fasttest because DEFLATE_QPL isn't available in fasttest - SET send_logs_level = 'fatal'; DROP TABLE IF EXISTS alter_compression_codec; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql index df74620a2011..89e77f758a7a 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql @@ -1,6 +1,3 @@ ---Tags: no-fasttest --- no-fasttest because DEFLATE_QPL isn't available in fasttest - SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; SET enable_deflate_qpl_codec = 1; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql index 67c0074c58fa..a629df2666d2 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql @@ -1,6 +1,3 @@ ---Tags: no-fasttest --- no-fasttest because DEFLATE_QPL isn't available in fasttest - SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; SET enable_deflate_qpl_codec = 1; diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql index a46272112a9c..5a56fc0d5762 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -1,6 +1,3 @@ ---Tags: no-fasttest --- no-fasttest because DEFLATE_QPL isn't available in fasttest - SET send_logs_level = 'fatal'; SET enable_deflate_qpl_codec = 1; From d228411f41eabf7e443fbbb2f4148880a3da78fa Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 11 Jun 2023 14:39:15 +0000 Subject: [PATCH 16/20] Reset modified tests --- ...04_test_alter_compression_codecs.reference | 17 ++----- .../00804_test_alter_compression_codecs.sql | 22 +++------- ...4_test_custom_compression_codecs.reference | 8 ++-- .../00804_test_custom_compression_codecs.sql | 44 ++++++++----------- ...m_compression_codes_log_storages.reference | 20 ++++----- ..._custom_compression_codes_log_storages.sql | 41 ++++++++--------- ...804_test_deflate_qpl_codec_compression.sql | 4 ++ 7 files changed, 63 insertions(+), 93 deletions(-) diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference b/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference index 5c77a102740d..cfbfadf1e676 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.reference @@ -18,7 +18,7 @@ CODEC(NONE) 2018-01-01 4 4 2018-01-01 5 5 2018-01-01 6 6 -CODEC(DEFLATE_QPL) +CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, NONE) 2018-01-01 1 default_value 2018-01-01 2 default_value 2018-01-01 3 3 @@ -27,18 +27,7 @@ CODEC(DEFLATE_QPL) 2018-01-01 6 6 2018-01-01 7 7 2018-01-01 8 8 -CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, DEFLATE_QPL, NONE) -2018-01-01 1 default_value -2018-01-01 2 default_value -2018-01-01 3 3 -2018-01-01 4 4 -2018-01-01 5 5 -2018-01-01 6 6 -2018-01-01 7 7 -2018-01-01 8 8 -2018-01-01 9 9 -2018-01-01 10 10 -CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, DEFLATE_QPL, NONE) -CODEC(NONE, LZ4, LZ4HC(0), ZSTD(1), DEFLATE_QPL) +CODEC(ZSTD(1), LZ4HC(0), LZ4, LZ4, NONE) +CODEC(NONE, LZ4, LZ4HC(0), ZSTD(1)) 2 1 diff --git a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql index fd9855e82d33..85e5f8b63ad6 100644 --- a/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_alter_compression_codecs.sql @@ -28,20 +28,12 @@ SELECT * FROM alter_compression_codec ORDER BY id; OPTIMIZE TABLE alter_compression_codec FINAL; SELECT * FROM alter_compression_codec ORDER BY id; -SET enable_deflate_qpl_codec = 1; -ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL); +SET allow_suspicious_codecs = 1; +ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(ZSTD, LZ4HC, LZ4, LZ4, NONE); SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; INSERT INTO alter_compression_codec VALUES('2018-01-01', 7, '7'); INSERT INTO alter_compression_codec VALUES('2018-01-01', 8, '8'); -SELECT * FROM alter_compression_codec ORDER BY id; - -SET allow_suspicious_codecs = 1; -ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(ZSTD, LZ4HC, LZ4, LZ4, DEFLATE_QPL, NONE); -SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; - -INSERT INTO alter_compression_codec VALUES('2018-01-01', 9, '9'); -INSERT INTO alter_compression_codec VALUES('2018-01-01', 10, '10'); OPTIMIZE TABLE alter_compression_codec FINAL; SELECT * FROM alter_compression_codec ORDER BY id; @@ -62,17 +54,15 @@ ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 00:00:00' CODEC(ZSTD(100)); -- { serverError 433 } -ALTER TABLE alter_bad_codec ADD COLUMN alter_column DateTime DEFAULT '2019-01-01 00:00:00' CODEC(DEFLATE_QPL(100)); -- { serverError DATA_TYPE_CANNOT_HAVE_ARGUMENTS } - DROP TABLE IF EXISTS alter_bad_codec; DROP TABLE IF EXISTS large_alter_table_00804; DROP TABLE IF EXISTS store_of_hash_00804; CREATE TABLE large_alter_table_00804 ( - somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), DEFLATE_QPL), - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, DEFLATE_QPL), - data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4, DEFLATE_QPL) + somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12)), + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC), + data String CODEC(ZSTD(2), LZ4HC, NONE, LZ4, LZ4) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi', min_bytes_for_wide_part = 0; INSERT INTO large_alter_table_00804 SELECT toDate('2019-01-01'), number, toString(number + rand()) FROM system.numbers LIMIT 300000; @@ -81,7 +71,7 @@ CREATE TABLE store_of_hash_00804 (hash UInt64) ENGINE = Memory(); INSERT INTO store_of_hash_00804 SELECT sum(cityHash64(*)) FROM large_alter_table_00804; -ALTER TABLE large_alter_table_00804 MODIFY COLUMN data CODEC(NONE, LZ4, LZ4HC, ZSTD, DEFLATE_QPL); +ALTER TABLE large_alter_table_00804 MODIFY COLUMN data CODEC(NONE, LZ4, LZ4HC, ZSTD); OPTIMIZE TABLE large_alter_table_00804; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference index 8b51d65004a3..7bd91e5a69b9 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference @@ -1,6 +1,6 @@ -1 hello 2018-12-14 2018-12-14 1.1 aaa 5 -2 world 2018-12-15 2018-12-15 2.2 bbb 6 -3 ! 2018-12-16 2018-12-16 3.3 ccc 7 +1 hello 2018-12-14 1.1 aaa 5 +2 world 2018-12-15 2.2 bbb 6 +3 ! 2018-12-16 3.3 ccc 7 2 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 @@ -9,7 +9,7 @@ 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0), DEFLATE_QPL),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), NONE, NONE, NONE, LZ4HC(0), DEFLATE_QPL),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8), DEFLATE_QPL),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8), DEFLATE_QPL)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 +CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] 7.1 xxxxxxxxxxxx [127] ['Henry'] ! diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql index 89e77f758a7a..c080c2fc98ee 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.sql @@ -1,6 +1,5 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; -SET enable_deflate_qpl_codec = 1; DROP TABLE IF EXISTS compression_codec; @@ -8,21 +7,20 @@ CREATE TABLE compression_codec( id UInt64 CODEC(LZ4), data String CODEC(ZSTD), ddd Date CODEC(NONE), - ddd32 Date32 CODEC(DEFLATE_QPL), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), - othernum Int64 CODEC(Delta), + othernum Int64 CODEC(Delta) ) ENGINE = MergeTree() ORDER BY tuple(); -INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), toDate32('2018-12-14'), 1.1, 'aaa', 5); -INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), toDate32('2018-12-15'), 2.2, 'bbb', 6); -INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), toDate32('2018-12-16'), 3.3, 'ccc', 7); +INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); +INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); +INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); SELECT * FROM compression_codec ORDER BY id; OPTIMIZE TABLE compression_codec FINAL; -INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), toDate32('2018-12-13'), 4.4, 'ddd', 8); +INSERT INTO compression_codec VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); DETACH TABLE compression_codec; ATTACH TABLE compression_codec; @@ -33,31 +31,25 @@ DROP TABLE IF EXISTS compression_codec; DROP TABLE IF EXISTS bad_codec; DROP TABLE IF EXISTS params_when_no_params; -DROP TABLE IF EXISTS params_when_no_params2; DROP TABLE IF EXISTS too_many_params; DROP TABLE IF EXISTS codec_multiple_direct_specification_1; DROP TABLE IF EXISTS codec_multiple_direct_specification_2; -DROP TABLE IF EXISTS codec_multiple_direct_specification_3; DROP TABLE IF EXISTS delta_bad_params1; DROP TABLE IF EXISTS delta_bad_params2; CREATE TABLE bad_codec(id UInt64 CODEC(adssadads)) ENGINE = MergeTree() order by tuple(); -- { serverError 432 } CREATE TABLE too_many_params(id UInt64 CODEC(ZSTD(2,3,4,5))) ENGINE = MergeTree() order by tuple(); -- { serverError 431 } CREATE TABLE params_when_no_params(id UInt64 CODEC(LZ4(1))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 378 } -CREATE TABLE params_when_no_params2(id UInt64 CODEC(DEFLATE_QPL(1))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 378 } CREATE TABLE codec_multiple_direct_specification_1(id UInt64 CODEC(MULTIPLE(LZ4, ZSTD))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 432 } CREATE TABLE codec_multiple_direct_specification_2(id UInt64 CODEC(multiple(LZ4, ZSTD))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 432 } -CREATE TABLE codec_multiple_direct_specification_3(id UInt64 CODEC(multiple(LZ4, DEFLATE_QPL))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 432 } CREATE TABLE delta_bad_params1(id UInt64 CODEC(Delta(3))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 433 } CREATE TABLE delta_bad_params2(id UInt64 CODEC(Delta(16))) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 433 } DROP TABLE IF EXISTS bad_codec; DROP TABLE IF EXISTS params_when_no_params; -DROP TABLE IF EXISTS params_when_no_params2; DROP TABLE IF EXISTS too_many_params; DROP TABLE IF EXISTS codec_multiple_direct_specification_1; DROP TABLE IF EXISTS codec_multiple_direct_specification_2; -DROP TABLE IF EXISTS codec_multiple_direct_specification_3; DROP TABLE IF EXISTS delta_bad_params1; DROP TABLE IF EXISTS delta_bad_params2; @@ -66,10 +58,10 @@ DROP TABLE IF EXISTS compression_codec_multiple; SET network_compression_method = 'lz4hc'; CREATE TABLE compression_codec_multiple ( - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4), DEFLATE_QPL), - data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8), DEFLATE_QPL), - ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC, DEFLATE_QPL), - somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD, DEFLATE_QPL) + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) ) ENGINE = MergeTree() ORDER BY tuple(); INSERT INTO compression_codec_multiple VALUES (1, 'world', toDate('2018-10-05'), 1.1), (2, 'hello', toDate('2018-10-01'), 2.2), (3, 'buy', toDate('2018-10-11'), 3.3); @@ -93,15 +85,15 @@ SELECT sum(cityHash64(*)) FROM compression_codec_multiple; DROP TABLE IF EXISTS compression_codec_multiple_more_types; CREATE TABLE compression_codec_multiple_more_types ( - id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC, DEFLATE_QPL), - data FixedString(12) CODEC(ZSTD, ZSTD, Delta, Delta, Delta, NONE, NONE, NONE, LZ4HC, DEFLATE_QPL), - ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8), DEFLATE_QPL) + id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC), + data FixedString(12) CODEC(ZSTD, ZSTD, Delta, Delta, Delta, NONE, NONE, NONE, LZ4HC), + ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8)) ) ENGINE = MergeTree() ORDER BY tuple(); -- { serverError 36 } CREATE TABLE compression_codec_multiple_more_types ( - id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC, DEFLATE_QPL), - data FixedString(12) CODEC(ZSTD, ZSTD, NONE, NONE, NONE, LZ4HC, DEFLATE_QPL), - ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8), DEFLATE_QPL) + id Decimal128(13) CODEC(ZSTD, LZ4, ZSTD, ZSTD, Delta(2), Delta(4), Delta(1), LZ4HC), + data FixedString(12) CODEC(ZSTD, ZSTD, NONE, NONE, NONE, LZ4HC), + ddd Nested (age UInt8, Name String) CODEC(LZ4, LZ4HC, NONE, NONE, NONE, ZSTD, Delta(8)) ) ENGINE = MergeTree() ORDER BY tuple(); SHOW CREATE TABLE compression_codec_multiple_more_types; @@ -117,9 +109,9 @@ SET network_compression_method = 'zstd'; SET network_zstd_compression_level = 5; CREATE TABLE compression_codec_multiple_with_key ( - somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), Delta, Delta, DEFLATE_QPL), - id UInt64 CODEC(LZ4, ZSTD, Delta, NONE, LZ4HC, Delta, DEFLATE_QPL), - data String CODEC(ZSTD(2), Delta(1), LZ4HC, NONE, LZ4, LZ4, DEFLATE_QPL) + somedate Date CODEC(ZSTD, ZSTD, ZSTD(12), LZ4HC(12), Delta, Delta), + id UInt64 CODEC(LZ4, ZSTD, Delta, NONE, LZ4HC, Delta), + data String CODEC(ZSTD(2), Delta(1), LZ4HC, NONE, LZ4, LZ4) ) ENGINE = MergeTree() PARTITION BY somedate ORDER BY id SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi'; diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference index d64b8a77eed0..8145ca998292 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference @@ -1,9 +1,9 @@ -CREATE TABLE default.compression_codec_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8)),\n `qplstr` String CODEC(DEFLATE_QPL),\n `qplnum` UInt32 CODEC(DEFLATE_QPL)\n)\nENGINE = Log -1 hello 2018-12-14 1.1 aaa 5 qpl11 11 -2 world 2018-12-15 2.2 bbb 6 qpl22 22 -3 ! 2018-12-16 3.3 ccc 7 qpl33 33 +CREATE TABLE default.compression_codec_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = Log +1 hello 2018-12-14 1.1 aaa 5 +2 world 2018-12-15 2.2 bbb 6 +3 ! 2018-12-16 3.3 ccc 7 2 -CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4), DEFLATE_QPL),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8), DEFLATE_QPL),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0), DEFLATE_QPL),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1), DEFLATE_QPL)\n)\nENGINE = Log +CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = Log 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 @@ -11,12 +11,12 @@ CREATE TABLE default.compression_codec_multiple_log\n(\n `id` UInt64 CODEC(LZ 10003 274972506.6 9175437371954010821 -CREATE TABLE default.compression_codec_tiny_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8)),\n `qplstr` String CODEC(DEFLATE_QPL),\n `qplnum` UInt32 CODEC(DEFLATE_QPL)\n)\nENGINE = TinyLog -1 hello 2018-12-14 1.1 aaa 5 qpl11 11 -2 world 2018-12-15 2.2 bbb 6 qpl22 22 -3 ! 2018-12-16 3.3 ccc 7 qpl33 33 +CREATE TABLE default.compression_codec_tiny_log\n(\n `id` UInt64 CODEC(LZ4),\n `data` String CODEC(ZSTD(1)),\n `ddd` Date CODEC(NONE),\n `somenum` Float64 CODEC(ZSTD(2)),\n `somestr` FixedString(3) CODEC(LZ4HC(7)),\n `othernum` Int64 CODEC(Delta(8))\n)\nENGINE = TinyLog +1 hello 2018-12-14 1.1 aaa 5 +2 world 2018-12-15 2.2 bbb 6 +3 ! 2018-12-16 3.3 ccc 7 2 -CREATE TABLE default.compression_codec_multiple_tiny_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4), DEFLATE_QPL),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8), DEFLATE_QPL),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0), DEFLATE_QPL),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1), DEFLATE_QPL)\n)\nENGINE = TinyLog +CREATE TABLE default.compression_codec_multiple_tiny_log\n(\n `id` UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)),\n `data` String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)),\n `ddd` Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)),\n `somenum` Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))\n)\nENGINE = TinyLog 1 world 2018-10-05 1.1 2 hello 2018-10-01 2.2 3 buy 2018-10-11 3.3 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql index a629df2666d2..fba6a2167627 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql +++ b/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql @@ -1,6 +1,5 @@ SET send_logs_level = 'fatal'; SET allow_suspicious_codecs = 1; -SET enable_deflate_qpl_codec = 1; -- copy-paste for storage log @@ -12,20 +11,18 @@ CREATE TABLE compression_codec_log( ddd Date CODEC(NONE), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), - othernum Int64 CODEC(Delta), - qplstr String CODEC(DEFLATE_QPL), - qplnum UInt32 CODEC(DEFLATE_QPL), + othernum Int64 CODEC(Delta) ) ENGINE = Log(); SHOW CREATE TABLE compression_codec_log; -INSERT INTO compression_codec_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); -INSERT INTO compression_codec_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6,'qpl22', 22); -INSERT INTO compression_codec_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); +INSERT INTO compression_codec_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); +INSERT INTO compression_codec_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); +INSERT INTO compression_codec_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); SELECT * FROM compression_codec_log ORDER BY id; -INSERT INTO compression_codec_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); +INSERT INTO compression_codec_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); DETACH TABLE compression_codec_log; ATTACH TABLE compression_codec_log; @@ -37,10 +34,10 @@ DROP TABLE IF EXISTS compression_codec_log; DROP TABLE IF EXISTS compression_codec_multiple_log; CREATE TABLE compression_codec_multiple_log ( - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4), DEFLATE_QPL), - data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8), DEFLATE_QPL), - ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC, DEFLATE_QPL), - somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD, DEFLATE_QPL) + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) ) ENGINE = Log(); SHOW CREATE TABLE compression_codec_multiple_log; @@ -72,20 +69,18 @@ CREATE TABLE compression_codec_tiny_log( ddd Date CODEC(NONE), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), - othernum Int64 CODEC(Delta), - qplstr String CODEC(DEFLATE_QPL), - qplnum UInt32 CODEC(DEFLATE_QPL), + othernum Int64 CODEC(Delta) ) ENGINE = TinyLog(); SHOW CREATE TABLE compression_codec_tiny_log; -INSERT INTO compression_codec_tiny_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5, 'qpl11', 11); -INSERT INTO compression_codec_tiny_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6, 'qpl22', 22); -INSERT INTO compression_codec_tiny_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7, 'qpl33', 33); +INSERT INTO compression_codec_tiny_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); +INSERT INTO compression_codec_tiny_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); +INSERT INTO compression_codec_tiny_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); SELECT * FROM compression_codec_tiny_log ORDER BY id; -INSERT INTO compression_codec_tiny_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8, 'qpl44', 44); +INSERT INTO compression_codec_tiny_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); DETACH TABLE compression_codec_tiny_log; ATTACH TABLE compression_codec_tiny_log; @@ -97,10 +92,10 @@ DROP TABLE IF EXISTS compression_codec_tiny_log; DROP TABLE IF EXISTS compression_codec_multiple_tiny_log; CREATE TABLE compression_codec_multiple_tiny_log ( - id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4), DEFLATE_QPL), - data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8), DEFLATE_QPL), - ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC, DEFLATE_QPL), - somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD, DEFLATE_QPL) + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) ) ENGINE = TinyLog(); SHOW CREATE TABLE compression_codec_multiple_tiny_log; diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql index 5a56fc0d5762..78c57013eeb2 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -1,3 +1,7 @@ +--Tags: no-fasttest, no-cpu-aarch64 +-- no-fasttest because DEFLATE_QPL isn't available in fasttest +-- no-cpu-aarch64 because DEFLATE_QPL is x86-only + SET send_logs_level = 'fatal'; SET enable_deflate_qpl_codec = 1; From 2419a7b90fd1effd8ebf8b5b4741a0325447cdec Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 11 Jun 2023 15:16:52 +0000 Subject: [PATCH 17/20] Fix tests --- .../00804_test_deflate_qpl_codec_compression.reference | 2 ++ .../00804_test_deflate_qpl_codec_compression.sql | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference index 276747f82336..a2178f5eda7e 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.reference @@ -1,4 +1,6 @@ +CREATE TABLE default.compression_codec\n(\n `id` UInt64 CODEC(DEFLATE_QPL),\n `data` String CODEC(DEFLATE_QPL),\n `ddd` Date CODEC(DEFLATE_QPL),\n `ddd32` Date32 CODEC(DEFLATE_QPL),\n `somenum` Float64 CODEC(DEFLATE_QPL),\n `somestr` FixedString(3) CODEC(DEFLATE_QPL),\n `othernum` Int64 CODEC(DEFLATE_QPL),\n `somearray` Array(UInt8) CODEC(DEFLATE_QPL),\n `somemap` Map(String, UInt32) CODEC(DEFLATE_QPL),\n `sometuple` Tuple(UInt16, UInt64) CODEC(DEFLATE_QPL)\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1 hello 2018-12-14 2018-12-14 1.1 aaa 5 [1,2,3] {'k1':1,'k2':2} (1,2) 2 world 2018-12-15 2018-12-15 2.2 bbb 6 [4,5,6] {'k3':3,'k4':4} (3,4) 3 ! 2018-12-16 2018-12-16 3.3 ccc 7 [7,8,9] {'k5':5,'k6':6} (5,6) 2 +10001 diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql index 78c57013eeb2..ff3c1812c86c 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -2,7 +2,8 @@ -- no-fasttest because DEFLATE_QPL isn't available in fasttest -- no-cpu-aarch64 because DEFLATE_QPL is x86-only -SET send_logs_level = 'fatal'; +-- A bunch of random DDLs to test the DEFLATE_QPL codec. + SET enable_deflate_qpl_codec = 1; DROP TABLE IF EXISTS compression_codec; @@ -20,6 +21,8 @@ CREATE TABLE compression_codec( sometuple Tuple(UInt16, UInt64) CODEC(DEFLATE_QPL), ) ENGINE = MergeTree() ORDER BY tuple(); +SHOW CREATE TABLE compression_codec; + INSERT INTO compression_codec VALUES(1, 'hello', toDate('2018-12-14'), toDate32('2018-12-14'), 1.1, 'aaa', 5, [1,2,3], map('k1',1,'k2',2), tuple(1,2)); INSERT INTO compression_codec VALUES(2, 'world', toDate('2018-12-15'), toDate32('2018-12-15'), 2.2, 'bbb', 6, [4,5,6], map('k3',3,'k4',4), tuple(3,4)); INSERT INTO compression_codec VALUES(3, '!', toDate('2018-12-16'), toDate32('2018-12-16'), 3.3, 'ccc', 7, [7,8,9], map('k5',5,'k6',6), tuple(5,6)); @@ -35,4 +38,8 @@ ATTACH TABLE compression_codec; SELECT count(*) FROM compression_codec WHERE id = 2 GROUP BY id; +INSERT INTO compression_codec SELECT 3, '!', toDate('2018-12-16'), toDate32('2018-12-16'), 3.3, 'ccc', 7, [7,8,9], map('k5',5,'k6',6), tuple(5,6) FROM system.numbers LIMIT 10000; + +SELECT count(*) FROM compression_codec WHERE id = 3 GROUP BY id; + DROP TABLE IF EXISTS compression_codec; From 1af062a53214168345838f79cba53ecb11cbc41e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 12 Jun 2023 08:04:55 +0000 Subject: [PATCH 18/20] Un-flake 00804_test_deflate_qpl_codec_compression --- .../0_stateless/00804_test_deflate_qpl_codec_compression.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql index ff3c1812c86c..8a256567e804 100644 --- a/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql +++ b/tests/queries/0_stateless/00804_test_deflate_qpl_codec_compression.sql @@ -6,6 +6,10 @@ SET enable_deflate_qpl_codec = 1; +-- Suppress test failures because stderr contains warning "Initialization of hardware-assisted DeflateQpl failed, falling +-- back to software DeflateQpl coded." +SET send_logs_level = 'fatal'; + DROP TABLE IF EXISTS compression_codec; CREATE TABLE compression_codec( From 5f8ef275139d77d66ba7b6a7b729528c43f4b1e2 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Fri, 16 Jun 2023 14:37:01 -0400 Subject: [PATCH 19/20] initialize buffer with memset --- src/Compression/CompressionCodecDeflateQpl.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Compression/CompressionCodecDeflateQpl.cpp b/src/Compression/CompressionCodecDeflateQpl.cpp index 60efbd96f8f6..7da79b5f90dd 100644 --- a/src/Compression/CompressionCodecDeflateQpl.cpp +++ b/src/Compression/CompressionCodecDeflateQpl.cpp @@ -382,6 +382,9 @@ UInt32 CompressionCodecDeflateQpl::getMaxCompressedDataSize(UInt32 uncompressed_ UInt32 CompressionCodecDeflateQpl::doCompressData(const char * source, UInt32 source_size, char * dest) const { +#if defined(MEMORY_SANITIZER) + memset(dest, 0 , getMaxCompressedDataSize(source_size)); +#endif Int32 res = HardwareCodecDeflateQpl::RET_ERROR; if (DeflateQplJobHWPool::instance().isJobPoolReady()) res = hw_codec->doCompressData(source, source_size, dest, getMaxCompressedDataSize(source_size)); @@ -392,6 +395,9 @@ UInt32 CompressionCodecDeflateQpl::doCompressData(const char * source, UInt32 so void CompressionCodecDeflateQpl::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { +#if defined(MEMORY_SANITIZER) + memset(dest, 0 , uncompressed_size); +#endif switch (getDecompressMode()) { case CodecMode::Synchronous: From 004755914ea60b9c84a9f2eb8e39c0f3613c8261 Mon Sep 17 00:00:00 2001 From: jinjunzh Date: Sun, 18 Jun 2023 20:55:10 -0400 Subject: [PATCH 20/20] change to use generic MSan API to bypass uninitialized memory check --- src/Compression/CompressionCodecDeflateQpl.cpp | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Compression/CompressionCodecDeflateQpl.cpp b/src/Compression/CompressionCodecDeflateQpl.cpp index 7da79b5f90dd..4a2866d152d5 100644 --- a/src/Compression/CompressionCodecDeflateQpl.cpp +++ b/src/Compression/CompressionCodecDeflateQpl.cpp @@ -8,6 +8,7 @@ #include #include #include "libaccel_config.h" +#include namespace DB { @@ -382,8 +383,10 @@ UInt32 CompressionCodecDeflateQpl::getMaxCompressedDataSize(UInt32 uncompressed_ UInt32 CompressionCodecDeflateQpl::doCompressData(const char * source, UInt32 source_size, char * dest) const { +/// QPL library is using AVX-512 with some shuffle operations. +/// Memory sanitizer don't understand if there was uninitialized memory in SIMD register but it was not used in the result of shuffle. #if defined(MEMORY_SANITIZER) - memset(dest, 0 , getMaxCompressedDataSize(source_size)); + __msan_unpoison(dest, getMaxCompressedDataSize(source_size)); #endif Int32 res = HardwareCodecDeflateQpl::RET_ERROR; if (DeflateQplJobHWPool::instance().isJobPoolReady()) @@ -395,8 +398,10 @@ UInt32 CompressionCodecDeflateQpl::doCompressData(const char * source, UInt32 so void CompressionCodecDeflateQpl::doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const { +/// QPL library is using AVX-512 with some shuffle operations. +/// Memory sanitizer don't understand if there was uninitialized memory in SIMD register but it was not used in the result of shuffle. #if defined(MEMORY_SANITIZER) - memset(dest, 0 , uncompressed_size); + __msan_unpoison(dest, uncompressed_size); #endif switch (getDecompressMode()) {