Release pull request for branch 20.2

contrib/openssl-cmake/CMakeLists.txt

@@ -91,6 +91,7 @@ elseif (ARCH_AARCH64)
     perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha1-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-armv8.S)
     perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-armv8.S)   # This is not a mistake
     perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/sha/asm/sha512-armv8.pl ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-armv8.S)
+    perl_generate_asm(${OPENSSL_SOURCE_DIR}/crypto/modes/asm/aes-gcm-armv8_64.pl ${OPENSSL_BINARY_DIR}/crypto/modes/asm/aes-gcm-armv8_64.S)
 
 endif ()
 
@@ -188,6 +189,7 @@ ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_buff.c
 ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_lbuf.c
 ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_nbio.c
 ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_null.c
+${OPENSSL_SOURCE_DIR}/crypto/bio/bf_prefix.c
 ${OPENSSL_SOURCE_DIR}/crypto/bio/bio_cb.c
 ${OPENSSL_SOURCE_DIR}/crypto/bio/bio_err.c
 ${OPENSSL_SOURCE_DIR}/crypto/bio/bio_lib.c
@@ -320,14 +322,15 @@ ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_check.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_depr.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_err.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_gen.c
+${OPENSSL_SOURCE_DIR}/crypto/dh/dh_group_params.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_kdf.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_key.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_lib.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_meth.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_pmeth.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_prn.c
 ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_rfc5114.c
-${OPENSSL_SOURCE_DIR}/crypto/dh/dh_rfc7919.c
+${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_aid.c
 ${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_ameth.c
 ${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_asn1.c
 ${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_depr.c
@@ -464,10 +467,10 @@ ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_md5.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_md5_sha1.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_mdc2.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_sha.c
+${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_ripemd.c
+${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_wp.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/m_null.c
-${OPENSSL_SOURCE_DIR}/crypto/evp/m_ripemd.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/m_sigver.c
-${OPENSSL_SOURCE_DIR}/crypto/evp/m_wp.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/mac_lib.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/mac_meth.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/names.c
@@ -486,6 +489,8 @@ ${OPENSSL_SOURCE_DIR}/crypto/evp/pkey_mac.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_fn.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_gn.c
 ${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_lib.c
+${OPENSSL_SOURCE_DIR}/crypto/evp/signature.c
+${OPENSSL_SOURCE_DIR}/crypto/ffc/ffc_params.c
 ${OPENSSL_SOURCE_DIR}/crypto/hmac/hm_ameth.c
 ${OPENSSL_SOURCE_DIR}/crypto/hmac/hmac.c
 ${OPENSSL_SOURCE_DIR}/crypto/idea/i_cbc.c
@@ -529,6 +534,7 @@ ${OPENSSL_SOURCE_DIR}/crypto/provider_conf.c
 ${OPENSSL_SOURCE_DIR}/crypto/provider_core.c
 ${OPENSSL_SOURCE_DIR}/crypto/provider_predefined.c
 ${OPENSSL_SOURCE_DIR}/crypto/sparse_array.c
+${OPENSSL_SOURCE_DIR}/crypto/self_test_core.c
 ${OPENSSL_SOURCE_DIR}/crypto/threads_none.c
 ${OPENSSL_SOURCE_DIR}/crypto/threads_pthread.c
 ${OPENSSL_SOURCE_DIR}/crypto/threads_win.c
@@ -673,8 +679,8 @@ ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_crypt.c
 ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_err.c
 ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_pmeth.c
 ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_sign.c
-${OPENSSL_SOURCE_DIR}/crypto/sm3/m_sm3.c
 ${OPENSSL_SOURCE_DIR}/crypto/sm3/sm3.c
+${OPENSSL_SOURCE_DIR}/crypto/sm3/legacy_sm3.c
 ${OPENSSL_SOURCE_DIR}/crypto/sm4/sm4.c
 ${OPENSSL_SOURCE_DIR}/crypto/srp/srp_lib.c
 ${OPENSSL_SOURCE_DIR}/crypto/srp/srp_vfy.c
@@ -779,7 +785,11 @@ ${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509.c
 ${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509a.c
 ${OPENSSL_SOURCE_DIR}/providers/implementations/asymciphers/rsa_enc.c
 ${OPENSSL_SOURCE_DIR}/providers/defltprov.c
+${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_null.c
 ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes.c
+${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.c
+${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha1_hw.c
+${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha256_hw.c
 ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ccm.c
 ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ccm_hw.c
 ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_gcm.c
@@ -944,7 +954,8 @@ elseif (ARCH_AARCH64)
         ${OPENSSL_BINARY_DIR}/crypto/sha/keccak1600-armv8.S
         ${OPENSSL_BINARY_DIR}/crypto/sha/sha1-armv8.S
         ${OPENSSL_BINARY_DIR}/crypto/sha/sha256-armv8.S
-        ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-armv8.S)
+        ${OPENSSL_BINARY_DIR}/crypto/sha/sha512-armv8.S
+        ${OPENSSL_BINARY_DIR}/crypto/modes/asm/aes-gcm-armv8_64.S)
 endif ()
 
 set(SSL_SRCS

contrib/openssl-cmake/linux_aarch64/include/openssl/configuration.h

@@ -27,12 +27,7 @@ extern "C" {
 
 # define OPENSSL_CONFIGURED_API 30000
 
-/// This fragment was edited to avoid dependency on "getrandom" function that is not available on old libc and old Linux kernels.
-/// The DEVRANDOM method is also good.
-
-//# ifndef OPENSSL_RAND_SEED_OS
-//#  define OPENSSL_RAND_SEED_OS
-//# endif
+#define OPENSSL_RAND_SEED_OS
 #define OPENSSL_RAND_SEED_DEVRANDOM
 
 # ifndef OPENSSL_THREADS

contrib/openssl-cmake/linux_x86_64/include/openssl/configuration.h

@@ -27,12 +27,7 @@ extern "C" {
 
 # define OPENSSL_CONFIGURED_API 30000
 
-/// This fragment was edited to avoid dependency on "getrandom" function that is not available on old libc and old Linux kernels.
-/// The DEVRANDOM method is also good.
-
-//# ifndef OPENSSL_RAND_SEED_OS
-//#  define OPENSSL_RAND_SEED_OS
-//# endif
+#define OPENSSL_RAND_SEED_OS
 #define OPENSSL_RAND_SEED_DEVRANDOM
 
 # ifndef OPENSSL_THREADS

contrib/simdjson-cmake/CMakeLists.txt

@@ -1,14 +1,13 @@
 set(SIMDJSON_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/simdjson/include")
 set(SIMDJSON_SRC_DIR "${SIMDJSON_INCLUDE_DIR}/../src")
 set(SIMDJSON_SRC
+    ${SIMDJSON_SRC_DIR}/document.cpp
+    ${SIMDJSON_SRC_DIR}/error.cpp
+    ${SIMDJSON_SRC_DIR}/implementation.cpp
     ${SIMDJSON_SRC_DIR}/jsonioutil.cpp
     ${SIMDJSON_SRC_DIR}/jsonminifier.cpp
-    ${SIMDJSON_SRC_DIR}/jsonparser.cpp
     ${SIMDJSON_SRC_DIR}/stage1_find_marks.cpp
     ${SIMDJSON_SRC_DIR}/stage2_build_tape.cpp
-    ${SIMDJSON_SRC_DIR}/parsedjson.cpp
-    ${SIMDJSON_SRC_DIR}/parsedjsoniterator.cpp
-    ${SIMDJSON_SRC_DIR}/simdjson.cpp
 )
 
 add_library(${SIMDJSON_LIBRARY} ${SIMDJSON_SRC})

dbms/cmake/version.cmake

@@ -2,10 +2,10 @@
 set(VERSION_REVISION 54432)
 set(VERSION_MAJOR 20)
 set(VERSION_MINOR 2)
-set(VERSION_PATCH 1)
-set(VERSION_GITHASH 4b9acaaa9099e71c36e5c818031149c5cba2bbdb)
-set(VERSION_DESCRIBE v20.2.1.1-prestable)
-set(VERSION_STRING 20.2.1.1)
+set(VERSION_PATCH 2)
+set(VERSION_GITHASH d93e7e5ccf8fcca724e917581b00bf569947fff9)
+set(VERSION_DESCRIBE v20.2.2.1-prestable)
+set(VERSION_STRING 20.2.2.1)
 # end of autochange
 
 set(VERSION_EXTRA "" CACHE STRING "")

dbms/src/Common/OptimizedRegularExpression.cpp

@@ -409,7 +409,10 @@ unsigned OptimizedRegularExpressionImpl<thread_safe>::match(const char * subject
     if (is_trivial)
     {
         if (required_substring.empty())
+        {
+            matches.emplace_back(Match{0, 0});
             return 1;
+        }
 
         const UInt8 * pos;
         if (is_case_insensitive)

dbms/src/Common/ProfilingScopedRWLock.h

@@ -7,21 +7,51 @@
 
 namespace DB
 {
+
+class ProfilingScopedWriteUnlocker;
+
 class ProfilingScopedWriteRWLock
 {
 public:
-    ProfilingScopedWriteRWLock(std::shared_mutex & rwl, ProfileEvents::Event event) :
+    friend class ProfilingScopedWriteUnlocker;
+
+    ProfilingScopedWriteRWLock(std::shared_mutex & rwl_, ProfileEvents::Event event_) :
         watch(),
-        scoped_write_lock(rwl)
+        event(event_),
+        scoped_write_lock(rwl_)
     {
         ProfileEvents::increment(event, watch.elapsed());
     }
 
 private:
     Stopwatch watch;
+    ProfileEvents::Event event;
     std::unique_lock<std::shared_mutex> scoped_write_lock;
 };
 
+/// Inversed RAII
+/// Used to unlock current writelock for various purposes.
+class ProfilingScopedWriteUnlocker
+{
+public:
+    ProfilingScopedWriteUnlocker() = delete;
+
+    ProfilingScopedWriteUnlocker(ProfilingScopedWriteRWLock & parent_lock_) : parent_lock(parent_lock_)
+    {
+        parent_lock.scoped_write_lock.unlock();
+    }
+
+    ~ProfilingScopedWriteUnlocker()
+    {
+        Stopwatch watch;
+        parent_lock.scoped_write_lock.lock();
+        ProfileEvents::increment(parent_lock.event, watch.elapsed());
+    }
+
+private:
+    ProfilingScopedWriteRWLock & parent_lock;
+};
+
 class ProfilingScopedReadRWLock
 {
 public:

dbms/src/DataStreams/BlockIO.h

@@ -18,8 +18,8 @@ struct BlockIO
     BlockIO(const BlockIO &) = default;
     ~BlockIO() = default;
 
-    /** process_list_entry should be destroyed after in and after out,
-      *  since in and out contain pointer to objects inside process_list_entry (query-level MemoryTracker for example),
+    /** process_list_entry should be destroyed after in, after out and after pipeline,
+      *  since in, out and pipeline contain pointer to objects inside process_list_entry (query-level MemoryTracker for example),
       *  which could be used before destroying of in and out.
       */
     std::shared_ptr<ProcessListEntry> process_list_entry;
@@ -56,6 +56,7 @@ struct BlockIO
 
         out.reset();
         in.reset();
+        pipeline = QueryPipeline();
         process_list_entry.reset();
 
         process_list_entry      = rhs.process_list_entry;

dbms/src/DataStreams/ConvertingBlockInputStream.cpp

@@ -98,6 +98,10 @@ Block ConvertingBlockInputStream::readImpl()
         return src;
 
     Block res = header.cloneEmpty();
+    /// This is important because header.cloneEmpty() doesn't copy info about aggregation bucket.
+    /// Otherwise information in buckets may be lost (and aggregation will return wrong result).
+    res.info = src.info;
+
     for (size_t res_pos = 0, size = conversion.size(); res_pos < size; ++res_pos)
     {
         const auto & src_elem = src.getByPosition(conversion[res_pos]);

dbms/src/Dictionaries/CacheDictionary.cpp

@@ -820,6 +820,9 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
 
     const auto now = std::chrono::system_clock::now();
 
+    /// Non const because it will be unlocked.
+    ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs};
+
     if (now > backoff_end_time.load())
     {
         try
@@ -832,13 +835,26 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
             }
 
             Stopwatch watch;
-            auto stream = source_ptr->loadIds(bunch_update_unit.getRequestedIds());
 
-            const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs};
+            /// To perform parallel loading.
+            BlockInputStreamPtr stream = nullptr;
+            {
+                ProfilingScopedWriteUnlocker unlocker(write_lock);
+                stream = source_ptr->loadIds(bunch_update_unit.getRequestedIds());
+            }
 
             stream->readPrefix();
-            while (const auto block = stream->read())
+
+            while (true)
             {
+                Block block;
+                {
+                    ProfilingScopedWriteUnlocker unlocker(write_lock);
+                    block = stream->read();
+                    if (!block)
+                        break;
+                }
+
                 const auto id_column = typeid_cast<const ColumnUInt64 *>(block.safeGetByPosition(0).column.get());
                 if (!id_column)
                     throw Exception{name + ": id column has type different from UInt64.", ErrorCodes::TYPE_MISMATCH};
@@ -907,8 +923,6 @@ void CacheDictionary::update(BunchUpdateUnit & bunch_update_unit) const
 
     size_t not_found_num = 0, found_num = 0;
 
-    const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs};
-
     /// Check which ids have not been found and require setting null_value
     for (const auto & id_found_pair : remaining_ids)
     {

dbms/src/Dictionaries/ClickHouseDictionarySource.cpp

@@ -68,7 +68,7 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(
     , query_builder{dict_struct, db, table, where, IdentifierQuotingStyle::Backticks}
     , sample_block{sample_block_}
     , context(context_)
-    , is_local{isLocalAddress({host, port}, context.getTCPPort())}
+    , is_local{isLocalAddress({host, port}, secure ? context.getTCPPortSecure().value_or(0) : context.getTCPPort())}
     , pool{is_local ? nullptr : createPool(host, port, secure, db, user, password)}
     , load_all_query{query_builder.composeLoadAllQuery()}
 {

dbms/src/Dictionaries/ComplexKeyCacheDictionary.h

@@ -626,7 +626,7 @@ class ComplexKeyCacheDictionary final : public IDictionaryBase
             on_key_not_found(key, cell_idx);
         }
 
-        ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, found_num);
+        ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedFound, found_num);
         ProfileEvents::increment(ProfileEvents::DictCacheKeysRequestedMiss, not_found_num);
     }
 

dbms/src/Disks/DiskLocal.cpp

@@ -11,6 +11,7 @@
 
 namespace DB
 {
+
 namespace ErrorCodes
 {
     extern const int UNKNOWN_ELEMENT_IN_CONFIG;
@@ -106,7 +107,11 @@ bool DiskLocal::tryReserve(UInt64 bytes)
 
 UInt64 DiskLocal::getTotalSpace() const
 {
-    auto fs = getStatVFS(disk_path);
+    struct statvfs fs;
+    if (name == "default") /// for default disk we get space from path/data/
+        fs = getStatVFS(disk_path + "data/");
+    else
+        fs = getStatVFS(disk_path);
     UInt64 total_size = fs.f_blocks * fs.f_bsize;
     if (total_size < keep_free_space_bytes)
         return 0;
@@ -117,7 +122,11 @@ UInt64 DiskLocal::getAvailableSpace() const
 {
     /// we use f_bavail, because part of b_free space is
     /// available for superuser only and for system purposes
-    auto fs = getStatVFS(disk_path);
+    struct statvfs fs;
+    if (name == "default") /// for default disk we get space from path/data/
+        fs = getStatVFS(disk_path + "data/");
+    else
+        fs = getStatVFS(disk_path);
     UInt64 total_size = fs.f_bavail * fs.f_bsize;
     if (total_size < keep_free_space_bytes)
         return 0;

dbms/src/Disks/DiskSpaceMonitor.cpp

@@ -12,6 +12,16 @@
 
 namespace DB
 {
+
+namespace ErrorCodes
+{
+    extern const int EXCESSIVE_ELEMENT_IN_CONFIG;
+    extern const int UNKNOWN_DISK;
+    extern const int UNKNOWN_POLICY;
+    extern const int LOGICAL_ERROR;
+}
+
+
 DiskSelector::DiskSelector(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const Context & context)
 {
     Poco::Util::AbstractConfiguration::Keys keys;
@@ -253,10 +263,10 @@ DiskPtr StoragePolicy::getAnyDisk() const
     /// StoragePolicy must contain at least one Volume
     /// Volume must contain at least one Disk
     if (volumes.empty())
-        throw Exception("StoragePolicy has no volumes. It's a bug.", ErrorCodes::NOT_ENOUGH_SPACE);
+        throw Exception("StoragePolicy has no volumes. It's a bug.", ErrorCodes::LOGICAL_ERROR);
 
     if (volumes[0]->disks.empty())
-        throw Exception("Volume '" + volumes[0]->getName() + "' has no disks. It's a bug.", ErrorCodes::NOT_ENOUGH_SPACE);
+        throw Exception("Volume '" + volumes[0]->getName() + "' has no disks. It's a bug.", ErrorCodes::LOGICAL_ERROR);
 
     return volumes[0]->disks[0];
 }