Merge remote-tracking branch 'upstream/backport/23.10/57352' into HEAD

.github/workflows/backport_branches.yml

@@ -34,12 +34,7 @@ jobs:
       - name: Python unit tests
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          echo "Testing the main ci directory"
-          python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
+          python3 -m unittest discover -s . -p '*_test.py'
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     needs: CheckLabels
@@ -75,7 +70,7 @@ jobs:
           name: changed_images_amd64
           path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json
   DockerHubPush:
-    needs: [DockerHubPushAmd64, DockerHubPushAarch64, PythonUnitTests]
+    needs: [DockerHubPushAmd64, DockerHubPushAarch64]
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Check out repository code

.github/workflows/master.yml

@@ -20,12 +20,7 @@ jobs:
       - name: Python unit tests
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          echo "Testing the main ci directory"
-          python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
+          python3 -m unittest discover -s . -p '*_test.py'
   DockerHubPushAarch64:
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
@@ -96,7 +91,7 @@ jobs:
       runner_type: style-checker
       run_command: |
           cd "$REPO_COPY/tests/ci"
-          python3 style_check.py --no-push
+          python3 style_check.py
   CompatibilityCheckX86:
     needs: [BuilderDebRelease]
     uses: ./.github/workflows/reusable_test.yml

.github/workflows/pull_request.yml

@@ -48,10 +48,10 @@ jobs:
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           echo "Testing the main ci directory"
-          python3 -m unittest discover -s . -p 'test_*.py'
+          python3 -m unittest discover -s . -p '*_test.py'
           for dir in *_lambda/; do
             echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
+            python3 -m unittest discover -s "$dir" -p '*_test.py'
           done
   DockerHubPushAarch64:
     needs: CheckLabels

.gitmodules

@@ -1,6 +1,3 @@
-# Please do not use 'branch = ...' tags with submodule entries. Such tags make updating submodules a
-# little bit more convenient but they do *not* specify the tracked submodule branch. Thus, they are
-# more confusing than useful.
 [submodule "contrib/zstd"]
 	path = contrib/zstd
 	url = https://github.com/facebook/zstd
@@ -354,6 +351,3 @@
 [submodule "contrib/aklomp-base64"]
 	path = contrib/aklomp-base64
 	url = https://github.com/aklomp/base64.git
-[submodule "contrib/pocketfft"]
-	path = contrib/pocketfft
-	url = https://github.com/mreineck/pocketfft.git

CMakeLists.txt

@@ -21,11 +21,8 @@ include (cmake/clang_tidy.cmake)
 include (cmake/git.cmake)
 include (cmake/utils.cmake)
 
-# This is needed to set up the CMAKE_INSTALL_BINDIR variable.
-include (GNUInstallDirs)
-
 # Ignore export() since we don't use it,
-# but it gets broken with global targets via link_libraries()
+# but it gets broken with a global targets via link_libraries()
 macro (export)
 endmacro ()
 
@@ -167,7 +164,7 @@ if (OS_LINUX)
     #   and whatever is poisoning it by LD_PRELOAD should not link to our symbols.
     # - The clickhouse-odbc-bridge and clickhouse-library-bridge binaries
     #   should not expose their symbols to ODBC drivers and libraries.
-    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic -Wl,--gc-sections")
+    set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-export-dynamic")
 endif ()
 
 if (OS_DARWIN)
@@ -190,10 +187,9 @@ if (NOT CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE")
     endif ()
 endif()
 
-if (NOT (SANITIZE_COVERAGE OR WITH_COVERAGE)
-    AND (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE"
-        OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
-        OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL"))
+if (CMAKE_BUILD_TYPE_UC STREQUAL "RELEASE"
+    OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO"
+    OR CMAKE_BUILD_TYPE_UC STREQUAL "MINSIZEREL")
     set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT ON)
 else()
     set (OMIT_HEAVY_DEBUG_SYMBOLS_DEFAULT OFF)
@@ -277,11 +273,6 @@ option (ENABLE_BUILD_PROFILING "Enable profiling of build time" OFF)
 if (ENABLE_BUILD_PROFILING)
      if (COMPILER_CLANG)
         set (COMPILER_FLAGS "${COMPILER_FLAGS} -ftime-trace")
-
-        if (LINKER_NAME MATCHES "lld")
-            set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--time-trace")
-            set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -Wl,--time-trace")
-        endif ()
      else ()
         message (${RECONFIGURE_MESSAGE_LEVEL} "Build profiling is only available with CLang")
      endif ()
@@ -295,6 +286,9 @@ set (CMAKE_C_STANDARD 11)
 set (CMAKE_C_EXTENSIONS ON) # required by most contribs written in C
 set (CMAKE_C_STANDARD_REQUIRED ON)
 
+# Compiler-specific coverage flags e.g. -fcoverage-mapping
+option(WITH_COVERAGE "Profile the resulting binary/binaries" OFF)
+
 if (COMPILER_CLANG)
     # Enable C++14 sized global deallocation functions. It should be enabled by setting -std=c++14 but I'm not sure.
     # See https://reviews.llvm.org/D112921
@@ -310,12 +304,18 @@ if (COMPILER_CLANG)
         set(BRANCHES_WITHIN_32B_BOUNDARIES "-mbranches-within-32B-boundaries")
         set(COMPILER_FLAGS "${COMPILER_FLAGS} ${BRANCHES_WITHIN_32B_BOUNDARIES}")
     endif()
+
+    if (WITH_COVERAGE)
+        set(COMPILER_FLAGS "${COMPILER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+        # If we want to disable coverage for specific translation units
+        set(WITHOUT_COVERAGE "-fno-profile-instr-generate -fno-coverage-mapping")
+    endif()
 endif ()
 
 set (COMPILER_FLAGS "${COMPILER_FLAGS}")
 
 # Our built-in unwinder only supports DWARF version up to 4.
-set (DEBUG_INFO_FLAGS "-g")
+set (DEBUG_INFO_FLAGS "-g -gdwarf-4")
 
 # Disable omit frame pointer compiler optimization using -fno-omit-frame-pointer
 option(DISABLE_OMIT_FRAME_POINTER "Disable omit frame pointer compiler optimization" OFF)
@@ -463,6 +463,14 @@ endif ()
 
 message (STATUS "Building for: ${CMAKE_SYSTEM} ${CMAKE_SYSTEM_PROCESSOR} ${CMAKE_LIBRARY_ARCHITECTURE}")
 
+include (GNUInstallDirs)
+
+# When testing for memory leaks with Valgrind, don't link tcmalloc or jemalloc.
+
+if (TARGET global-group)
+    install (EXPORT global DESTINATION cmake)
+endif ()
+
 add_subdirectory (contrib EXCLUDE_FROM_ALL)
 
 if (NOT ENABLE_JEMALLOC)
@@ -546,16 +554,10 @@ if (ENABLE_RUST)
     endif()
 endif()
 
-if (CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO" AND NOT SANITIZE AND OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64))
-    set(CHECK_LARGE_OBJECT_SIZES_DEFAULT ON)
-else ()
-    set(CHECK_LARGE_OBJECT_SIZES_DEFAULT OFF)
-endif ()
-option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files after build." ${CHECK_LARGE_OBJECT_SIZES_DEFAULT})
-
 add_subdirectory (base)
 add_subdirectory (src)
 add_subdirectory (programs)
+add_subdirectory (tests)
 add_subdirectory (utils)
 
 if (FUZZER)

README.md

@@ -1,17 +1,6 @@
-<div align=center>
+[<img alt="ClickHouse — open source distributed column-oriented DBMS" width="400px" src="https://clickhouse.com/images/ch_gh_logo_rounded.png" />](https://clickhouse.com?utm_source=github)
 
-[![Website](https://img.shields.io/website?up_message=AVAILABLE&down_message=DOWN&url=https%3A%2F%2Fclickhouse.com&style=for-the-badge)](https://clickhouse.com)
-[![Apache 2.0 License](https://img.shields.io/badge/license-Apache%202.0-blueviolet?style=for-the-badge)](https://www.apache.org/licenses/LICENSE-2.0)
-
-<picture align=center>
-    <source media="(prefers-color-scheme: dark)" srcset="https://github.com/ClickHouse/clickhouse-docs/assets/9611008/4ef9c104-2d3f-4646-b186-507358d2fe28">
-    <source media="(prefers-color-scheme: light)" srcset="https://github.com/ClickHouse/clickhouse-docs/assets/9611008/b001dc7b-5a45-4dcd-9275-e03beb7f9177">
-    <img alt="The ClickHouse company logo." src="https://github.com/ClickHouse/clickhouse-docs/assets/9611008/b001dc7b-5a45-4dcd-9275-e03beb7f9177">
-</picture>
-
-<h4>ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.</h4>
-
-</div>
+ClickHouse® is an open-source column-oriented database management system that allows generating analytical data reports in real-time.
 
 ## How To Install (Linux, macOS, FreeBSD)
 ```
@@ -33,6 +22,11 @@ curl https://clickhouse.com/ | sh
 
 ## Upcoming Events
 
+* [**v23.9 Community Call**]([https://clickhouse.com/company/events/v23-8-community-release-call](https://clickhouse.com/company/events/v23-9-community-release-call)?utm_source=github&utm_medium=social&utm_campaign=release-webinar-2023-08) - Sep 28 - 23.9 is rapidly approaching. Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release.
+* [**ClickHouse Meetup in Amsterdam**](https://www.meetup.com/clickhouse-netherlands-user-group/events/296334590/) - Oct 31
+* [**ClickHouse Meetup in Beijing**](https://www.meetup.com/clickhouse-beijing-user-group/events/296334856/) - Nov 4
+* [**ClickHouse Meetup in San Francisco**](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/296334923/) - Nov 8
+* [**ClickHouse Meetup in Singapore**](https://www.meetup.com/clickhouse-singapore-meetup-group/events/296334976/) - Nov 15
 * [**ClickHouse Meetup in Berlin**](https://www.meetup.com/clickhouse-berlin-user-group/events/296488501/) - Nov 30
 * [**ClickHouse Meetup in NYC**](https://www.meetup.com/clickhouse-new-york-user-group/events/296488779/) - Dec 11
 * [**ClickHouse Meetup in Boston**](https://www.meetup.com/clickhouse-boston-user-group/events/296488840/) - Dec 12
@@ -41,7 +35,7 @@ Also, keep an eye out for upcoming meetups around the world. Somewhere else you
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
-* **Recording available**: [**v23.10 Release Webinar**](https://www.youtube.com/watch?v=PGQS6uPb970) All the features of 23.10, one convenient video! Watch it now!
+* **Recording available**: [**v23.6 Release Webinar**](https://www.youtube.com/watch?v=cuf_hYn7dqU) All the features of 23.6, one convenient video! Watch it now!
 * **All release webinar recordings**: [YouTube playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3jAlSy1JxyP8zluvXaN3nxU)
 
 

SECURITY.md

@@ -13,10 +13,9 @@ The following versions of ClickHouse server are currently being supported with s
 
 | Version | Supported |
 |:-|:-|
-| 23.10 | ✔️ |
 | 23.9 | ✔️ |
 | 23.8 | ✔️ |
-| 23.7 | ❌ |
+| 23.7 | ✔️ |
 | 23.6 | ❌ |
 | 23.5 | ❌ |
 | 23.4 | ❌ |

base/base/CMakeLists.txt

@@ -1,5 +1,3 @@
-add_compile_options($<$<OR:$<COMPILE_LANGUAGE:C>,$<COMPILE_LANGUAGE:CXX>>:${COVERAGE_FLAGS}>)
-
 if (USE_CLANG_TIDY)
     set (CMAKE_CXX_CLANG_TIDY "${CLANG_TIDY_PATH}")
 endif ()

base/base/coverage.cpp

@@ -1,15 +1,11 @@
 #include "coverage.h"
 
-#pragma GCC diagnostic ignored "-Wreserved-identifier"
-
-
-/// WITH_COVERAGE enables the default implementation of code coverage,
-/// that dumps a map to the filesystem.
-
 #if WITH_COVERAGE
 
-#include <mutex>
-#include <unistd.h>
+#pragma GCC diagnostic ignored "-Wreserved-identifier"
+
+#    include <mutex>
+#    include <unistd.h>
 
 
 #    if defined(__clang__)
@@ -35,131 +31,3 @@ void dumpCoverageReportIfPossible()
 
 #endif
 }
-
-
-/// SANITIZE_COVERAGE enables code instrumentation,
-/// but leaves the callbacks implementation to us,
-/// which we use to calculate coverage on a per-test basis
-/// and to write it to system tables.
-
-#if defined(SANITIZE_COVERAGE)
-
-namespace
-{
-    bool pc_guards_initialized = false;
-    bool pc_table_initialized = false;
-
-    uint32_t * guards_start = nullptr;
-    uint32_t * guards_end = nullptr;
-
-    uintptr_t * coverage_array = nullptr;
-    size_t coverage_array_size = 0;
-
-    uintptr_t * all_addresses_array = nullptr;
-    size_t all_addresses_array_size = 0;
-}
-
-extern "C"
-{
-
-/// This is called at least once for every DSO for initialization.
-/// But we will use it only for the main DSO.
-void __sanitizer_cov_trace_pc_guard_init(uint32_t * start, uint32_t * stop)
-{
-    if (pc_guards_initialized)
-        return;
-    pc_guards_initialized = true;
-
-    /// The function can be called multiple times, but we need to initialize only once.
-    if (start == stop || *start)
-        return;
-
-    guards_start = start;
-    guards_end = stop;
-    coverage_array_size = stop - start;
-
-    /// Note: we will leak this.
-    coverage_array = static_cast<uintptr_t*>(malloc(sizeof(uintptr_t) * coverage_array_size));
-
-    resetCoverage();
-}
-
-/// This is called at least once for every DSO for initialization
-/// and provides information about all instrumented addresses.
-void __sanitizer_cov_pcs_init(const uintptr_t * pcs_begin, const uintptr_t * pcs_end)
-{
-    if (pc_table_initialized)
-        return;
-    pc_table_initialized = true;
-
-    all_addresses_array = static_cast<uintptr_t*>(malloc(sizeof(uintptr_t) * coverage_array_size));
-    all_addresses_array_size = pcs_end - pcs_begin;
-
-    /// They are not a real pointers, but also contain a flag in the most significant bit,
-    /// in which we are not interested for now. Reset it.
-    for (size_t i = 0; i < all_addresses_array_size; ++i)
-        all_addresses_array[i] = pcs_begin[i] & 0x7FFFFFFFFFFFFFFFULL;
-}
-
-/// This is called at every basic block / edge, etc.
-void __sanitizer_cov_trace_pc_guard(uint32_t * guard)
-{
-    /// Duplicate the guard check.
-    if (!*guard)
-        return;
-    *guard = 0;
-
-    /// If you set *guard to 0 this code will not be called again for this edge.
-    /// Now we can get the PC and do whatever you want:
-    /// - store it somewhere or symbolize it and print right away.
-    /// The values of `*guard` are as you set them in
-    /// __sanitizer_cov_trace_pc_guard_init and so you can make them consecutive
-    /// and use them to dereference an array or a bit vector.
-    void * pc = __builtin_return_address(0);
-
-    coverage_array[guard - guards_start] = reinterpret_cast<uintptr_t>(pc);
-}
-
-}
-
-__attribute__((no_sanitize("coverage")))
-std::span<const uintptr_t> getCoverage()
-{
-    return {coverage_array, coverage_array_size};
-}
-
-__attribute__((no_sanitize("coverage")))
-std::span<const uintptr_t> getAllInstrumentedAddresses()
-{
-    return {all_addresses_array, all_addresses_array_size};
-}
-
-__attribute__((no_sanitize("coverage")))
-void resetCoverage()
-{
-    memset(coverage_array, 0, coverage_array_size * sizeof(*coverage_array));
-
-    /// The guard defines whether the __sanitizer_cov_trace_pc_guard should be called.
-    /// For example, you can unset it after first invocation to prevent excessive work.
-    /// Initially set all the guards to 1 to enable callbacks.
-    for (uint32_t * x = guards_start; x < guards_end; ++x)
-        *x = 1;
-}
-
-#else
-
-std::span<const uintptr_t> getCoverage()
-{
-    return {};
-}
-
-std::span<const uintptr_t> getAllInstrumentedAddresses()
-{
-    return {};
-}
-
-void resetCoverage()
-{
-}
-
-#endif