Merge branch 'master' into ldap-any-user-authentication

* master: (43 commits) Publish list of tests that failed the concurrent fast test fix test Use gnu++2a instead of c++2a for unbundled build to fix numeric_limits<__int128> fix test Update 01453_fixsed_string_sort.sql Added test. Fix ColumnString::updatePermutationWithCollation. Add support for extended precision integers and decimals (ClickHouse#13097) remove retries Check that ya.make files are auto-generated Fix "Arcadia" ISSUES-4006 trigger CI again fixed typo arrayCompact Stratify nans comparison in arrayCompact function Remove even more useless code Remove useless code around zkutil Fix 00956_sensitive_data_masking flackiness Update docker/test/stress/run.sh add exclusive DDLGuard for database Fix handling embedded config. ...
traceon · Aug 20, 2020 · cbc9285 · cbc9285
2 parents bdfea65 + 859eaf9
commit cbc9285
Show file tree

Hide file tree

Showing 171 changed files with 3,352 additions and 1,263 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -202,9 +202,16 @@ if (ARCH_NATIVE)
     set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=native")
 endif ()
 
-# cmake < 3.12 doesn't supoprt 20. We'll set CMAKE_CXX_FLAGS for now
+if (UNBUNDLED AND (COMPILER_GCC OR COMPILER_CLANG))
+    # to make numeric_limits<__int128> works for unbundled build
+    set (_CXX_STANDARD "-std=gnu++2a")
+else()
+    set (_CXX_STANDARD "-std=c++2a")
+endif()
+# cmake < 3.12 doesn't support 20. We'll set CMAKE_CXX_FLAGS for now
 # set (CMAKE_CXX_STANDARD 20)
-set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++2a")
+set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_CXX_STANDARD}")
+
 set (CMAKE_CXX_EXTENSIONS 0) # https://cmake.org/cmake/help/latest/prop_tgt/CXX_EXTENSIONS.html#prop_tgt:CXX_EXTENSIONS
 set (CMAKE_CXX_STANDARD_REQUIRED ON)
 

diff --git a/base/common/arithmeticOverflow.h b/base/common/arithmeticOverflow.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <common/types.h>
+
 namespace common
 {
     template <typename T>
@@ -35,6 +37,21 @@ namespace common
         return (y > 0 && x > max_int128 - y) || (y < 0 && x < min_int128 - y);
     }
 
+    template <>
+    inline bool addOverflow(bInt256 x, bInt256 y, bInt256 & res)
+    {
+        res = x + y;
+        return (y > 0 && x > std::numeric_limits<bInt256>::max() - y) ||
+            (y < 0 && x < std::numeric_limits<bInt256>::min() - y);
+    }
+
+    template <>
+    inline bool addOverflow(bUInt256 x, bUInt256 y, bUInt256 & res)
+    {
+        res = x + y;
+        return x > std::numeric_limits<bUInt256>::max() - y;
+    }
+
     template <typename T>
     inline bool subOverflow(T x, T y, T & res)
     {
@@ -68,6 +85,21 @@ namespace common
         return (y < 0 && x > max_int128 + y) || (y > 0 && x < min_int128 + y);
     }
 
+    template <>
+    inline bool subOverflow(bInt256 x, bInt256 y, bInt256 & res)
+    {
+        res = x - y;
+        return (y < 0 && x > std::numeric_limits<bInt256>::max() + y) ||
+            (y > 0 && x < std::numeric_limits<bInt256>::min() + y);
+    }
+
+    template <>
+    inline bool subOverflow(bUInt256 x, bUInt256 y, bUInt256 & res)
+    {
+        res = x - y;
+        return x < y;
+    }
+
     template <typename T>
     inline bool mulOverflow(T x, T y, T & res)
     {
@@ -103,4 +135,25 @@ namespace common
         unsigned __int128 b = (y > 0) ? y : -y;
         return (a * b) / b != a;
     }
+
+    template <>
+    inline bool mulOverflow(bInt256 x, bInt256 y, bInt256 & res)
+    {
+        res = x * y;
+        if (!x || !y)
+            return false;
+
+        bInt256 a = (x > 0) ? x : -x;
+        bInt256 b = (y > 0) ? y : -y;
+        return (a * b) / b != a;
+    }
+
+    template <>
+    inline bool mulOverflow(bUInt256 x, bUInt256 y, bUInt256 & res)
+    {
+        res = x * y;
+        if (!x || !y)
+            return false;
+        return (x * y) / y != x;
+    }
 }
diff --git a/base/common/tests/CMakeLists.txt b/base/common/tests/CMakeLists.txt
@@ -1,19 +1,15 @@
 include (${ClickHouse_SOURCE_DIR}/cmake/add_check.cmake)
 
-add_executable (date_lut_init date_lut_init.cpp)
 add_executable (date_lut2 date_lut2.cpp)
 add_executable (date_lut3 date_lut3.cpp)
-add_executable (date_lut4 date_lut4.cpp)
 add_executable (date_lut_default_timezone date_lut_default_timezone.cpp)
 add_executable (local_date_time_comparison local_date_time_comparison.cpp)
 add_executable (realloc-perf allocator.cpp)
 
 set(PLATFORM_LIBS ${CMAKE_DL_LIBS})
 
-target_link_libraries (date_lut_init PRIVATE common ${PLATFORM_LIBS})
 target_link_libraries (date_lut2 PRIVATE common ${PLATFORM_LIBS})
 target_link_libraries (date_lut3 PRIVATE common ${PLATFORM_LIBS})
-target_link_libraries (date_lut4 PRIVATE common ${PLATFORM_LIBS})
 target_link_libraries (date_lut_default_timezone PRIVATE common ${PLATFORM_LIBS})
 target_link_libraries (local_date_time_comparison PRIVATE common)
 target_link_libraries (realloc-perf PRIVATE common)

diff --git a/base/common/tests/date_lut4.cpp b/base/common/tests/date_lut4.cpp
diff --git a/base/common/tests/date_lut_init.cpp b/base/common/tests/date_lut_init.cpp
diff --git a/base/common/types.h b/base/common/types.h
@@ -6,6 +6,8 @@
 #include <string>
 #include <type_traits>
 
+#include <boost/multiprecision/cpp_int.hpp>
+
 using Int8 = int8_t;
 using Int16 = int16_t;
 using Int32 = int32_t;
@@ -15,11 +17,21 @@ using Int64 = int64_t;
 using char8_t = unsigned char;
 #endif
 
+/// This is needed for more strict aliasing. https://godbolt.org/z/xpJBSb https://stackoverflow.com/a/57453713
 using UInt8 = char8_t;
 using UInt16 = uint16_t;
 using UInt32 = uint32_t;
 using UInt64 = uint64_t;
 
+using Int128 = __int128;
+
+/// We have to use 127 and 255 bit integers to safe a bit for a sign serialization
+//using bInt256 = boost::multiprecision::int256_t;
+using bInt256 = boost::multiprecision::number<boost::multiprecision::cpp_int_backend<
+    255, 255, boost::multiprecision::signed_magnitude, boost::multiprecision::unchecked, void> >;
+using bUInt256 = boost::multiprecision::uint256_t;
+
+
 using String = std::string;
 
 /// The standard library type traits, such as std::is_arithmetic, with one exception
@@ -31,6 +43,9 @@ struct is_signed
     static constexpr bool value = std::is_signed_v<T>;
 };
 
+template <> struct is_signed<Int128> { static constexpr bool value = true; };
+template <> struct is_signed<bInt256> { static constexpr bool value = true; };
+
 template <typename T>
 inline constexpr bool is_signed_v = is_signed<T>::value;
 
@@ -40,23 +55,69 @@ struct is_unsigned
     static constexpr bool value = std::is_unsigned_v<T>;
 };
 
+template <> struct is_unsigned<bUInt256> { static constexpr bool value = true; };
+
 template <typename T>
 inline constexpr bool is_unsigned_v = is_unsigned<T>::value;
 
+
+/// TODO: is_integral includes char, char8_t and wchar_t.
 template <typename T>
-struct is_integral
+struct is_integer
 {
     static constexpr bool value = std::is_integral_v<T>;
 };
 
+template <> struct is_integer<Int128> { static constexpr bool value = true; };
+template <> struct is_integer<bInt256> { static constexpr bool value = true; };
+template <> struct is_integer<bUInt256> { static constexpr bool value = true; };
+
 template <typename T>
-inline constexpr bool is_integral_v = is_integral<T>::value;
+inline constexpr bool is_integer_v = is_integer<T>::value;
+
 
 template <typename T>
 struct is_arithmetic
 {
     static constexpr bool value = std::is_arithmetic_v<T>;
 };
 
+template <> struct is_arithmetic<__int128> { static constexpr bool value = true; };
+
 template <typename T>
 inline constexpr bool is_arithmetic_v = is_arithmetic<T>::value;
+
+template <typename T>
+struct make_unsigned
+{
+    typedef std::make_unsigned_t<T> type;
+};
+
+template <> struct make_unsigned<__int128> { using type = unsigned __int128; };
+template <> struct make_unsigned<bInt256>  { using type = bUInt256; };
+template <> struct make_unsigned<bUInt256> { using type = bUInt256; };
+
+template <typename T> using make_unsigned_t = typename make_unsigned<T>::type;
+
+template <typename T>
+struct make_signed
+{
+    typedef std::make_signed_t<T> type;
+};
+
+template <> struct make_signed<bInt256>  { typedef bInt256 type; };
+template <> struct make_signed<bUInt256> { typedef bInt256 type; };
+
+template <typename T> using make_signed_t = typename make_signed<T>::type;
+
+template <typename T>
+struct is_big_int
+{
+    static constexpr bool value = false;
+};
+
+template <> struct is_big_int<bUInt256> { static constexpr bool value = true; };
+template <> struct is_big_int<bInt256> { static constexpr bool value = true; };
+
+template <typename T>
+inline constexpr bool is_big_int_v = is_big_int<T>::value;
diff --git a/base/common/ya.make b/base/common/ya.make
@@ -25,8 +25,8 @@ PEERDIR(
     contrib/libs/cctz/src
     contrib/libs/cxxsupp/libcxx-filesystem
     contrib/libs/poco/Net
-    contrib/libs/poco/NetSSL_OpenSSL
     contrib/libs/poco/Util
+    contrib/libs/poco/NetSSL_OpenSSL
     contrib/libs/fmt
     contrib/restricted/boost
     contrib/restricted/cityhash-1.0.2
@@ -52,6 +52,7 @@ SRCS(
     shift10.cpp
     sleep.cpp
     terminalColors.cpp
+
 )
 
 END()
diff --git a/base/common/ya.make.in b/base/common/ya.make.in
@@ -10,6 +10,7 @@ CFLAGS (GLOBAL -DARCADIA_BUILD)
 CFLAGS (GLOBAL -DUSE_CPUID=1)
 CFLAGS (GLOBAL -DUSE_JEMALLOC=0)
 CFLAGS (GLOBAL -DUSE_RAPIDJSON=1)
+CFLAGS (GLOBAL -DUSE_SSL=1)
 
 IF (OS_DARWIN)
     CFLAGS (GLOBAL -DOS_DARWIN)
@@ -24,6 +25,7 @@ PEERDIR(
     contrib/libs/cxxsupp/libcxx-filesystem
     contrib/libs/poco/Net
     contrib/libs/poco/Util
+    contrib/libs/poco/NetSSL_OpenSSL
     contrib/libs/fmt
     contrib/restricted/boost
     contrib/restricted/cityhash-1.0.2

diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile
@@ -11,41 +11,42 @@ RUN echo "deb [trusted=yes] http://apt.llvm.org/eoan/ llvm-toolchain-eoan-10 mai
 
 
 RUN apt-get --allow-unauthenticated update -y \
-  && env DEBIAN_FRONTEND=noninteractive \
-      apt-get --allow-unauthenticated install --yes --no-install-recommends \
-          bash \
-          fakeroot \
-          ccache \
-          software-properties-common \
-          apt-transport-https \
-          ca-certificates \
-          wget \
-          bash \
-          fakeroot \
-          cmake \
-          ccache \
-          llvm-10 \
-          clang-10 \
-          lld-10 \
-          clang-tidy-10 \
-          ninja-build \
-          gperf \
-          git \
-          tzdata \
-          gperf \
-          rename \
-          build-essential \
-          expect \
-          python \
-          python-lxml \
-          python-termcolor \
-          python-requests \
-          unixodbc \
-          qemu-user-static \
-          sudo \
-          moreutils \
-          curl \
-          brotli
+    && env DEBIAN_FRONTEND=noninteractive \
+        apt-get --allow-unauthenticated install --yes --no-install-recommends \
+            apt-transport-https \
+            bash \
+            bash \
+            brotli \
+            build-essential \
+            ca-certificates \
+            ccache \
+            ccache \
+            clang-10 \
+            clang-tidy-10 \
+            cmake \
+            curl \
+            expect \
+            fakeroot \
+            fakeroot \
+            git \
+            gperf \
+            gperf \
+            lld-10 \
+            llvm-10 \
+            moreutils \
+            ninja-build \
+            psmisc \
+            python \
+            python-lxml \
+            python-requests \
+            python-termcolor \
+            qemu-user-static \
+            rename \
+            software-properties-common \
+            sudo \
+            tzdata \
+            unixodbc \
+            wget
 
 RUN mkdir -p /tmp/clickhouse-odbc-tmp \
   && wget --quiet -O - ${odbc_driver_url} | tar --strip-components=1 -xz -C /tmp/clickhouse-odbc-tmp \