Merge branch 'master' into ldap-per-user-authentication

* master: (1102 commits) Update README.md Update README.md Update README.md Update index.md [docs] add intrdocution for statements page (ClickHouse#12189) Revert "Run perf tests with memory sampling (for allocations >1M)" Sanitize LINK_LIBRARIES property for the directories (ClickHouse#12160) [docs] refactor Domains overview (ClickHouse#12186) DOCS-647: toStartOfSecond (ClickHouse#12190) [docs] add intrdocution for commercial page (ClickHouse#12187) DOCSUP-1348 Russian translation for new functions (#133) (ClickHouse#12194) changelog fixes Update index.md (ClickHouse#12191) Update zh kafka.md title (ClickHouse#12192) Added test for #3767 style fix for ClickHouse#12152 Tests for fixed issues ClickHouse#10846 and ClickHouse#7347 changelog fixes [docs] introduction for special table engines (ClickHouse#12170) [docs] introduction for third-party interfaces (ClickHouse#12175) ... # Conflicts: # src/Access/ya.make # src/Common/ErrorCodes.cpp
traceon · Jul 7, 2020 · 9effacf · 9effacf
2 parents 45d8199 + 546fcc9
commit 9effacf
Show file tree

Hide file tree

Showing 1,411 changed files with 42,631 additions and 9,826 deletions.
diff --git a/.github/workflows/anchore-analysis.yml b/.github/workflows/anchore-analysis.yml
@@ -0,0 +1,45 @@
+# This workflow checks out code, performs an Anchore container image
+# vulnerability and compliance scan, and integrates the results with
+# GitHub Advanced Security code scanning feature.  For more information on
+# the Anchore scan action usage and parameters, see
+# https://github.com/anchore/scan-action.  For more information on
+# Anchore container image scanning in general, see
+# https://docs.anchore.com.
+
+name: Docker Container Scan (clickhouse-server)
+
+on:
+  pull_request:
+    paths: 
+      - docker/server/Dockerfile
+      - .github/workflows/anchore-analysis.yml
+  schedule:
+    - cron: '0 21 * * *'
+
+jobs:
+  Anchore-Build-Scan:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout the code
+      uses: actions/checkout@v2
+    - name: Build the Docker image
+      run: |
+        cd docker/server
+        perl -pi -e 's|=\$version||g' Dockerfile
+        docker build . --file Dockerfile --tag localbuild/testimage:latest      
+    - name: Run the local Anchore scan action itself with GitHub Advanced Security code scanning integration enabled
+      uses: anchore/scan-action@master
+      with:
+        image-reference: "localbuild/testimage:latest"
+        dockerfile-path: "docker/server/Dockerfile"
+        acs-report-enable: true
+        fail-build: true
+    - name: Upload artifact
+      uses: actions/upload-artifact@v1.0.0
+      with:
+        name: AnchoreReports
+        path: ./anchore-reports/
+    - name: Upload Anchore Scan Report
+      uses: github/codeql-action/upload-sarif@v1
+      with:
+        sarif_file: results.sarif
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,33 @@
+name: "CodeQL Scanning"
+
+on:
+  schedule:
+    - cron: '0 19 * * *'
+jobs:
+  CodeQL-Build:
+
+    runs-on: self-hosted
+    timeout-minutes: 1440
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 2
+        submodules: 'recursive'
+
+    - run: git checkout HEAD^2
+      if: ${{ github.event_name == 'pull_request' }}
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+
+      with:
+        languages: cpp
+
+    - run: sudo apt-get update && sudo apt-get install -y git cmake python ninja-build gcc-9 g++-9 && mkdir build
+    - run: cd build && CC=gcc-9 CXX=g++-9 cmake ..
+    - run: cd build && ninja
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1
diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@
 /build
 /build_*
 /build-*
+/tests/venv
 
 /docs/build
 /docs/publish

diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,6 @@
 [submodule "contrib/poco"]
 	path = contrib/poco
-	url = https://github.com/ClickHouse-Extras/poco
+	url = https://github.com/ClickHouse-Extras/poco.git
 	branch = clickhouse
 [submodule "contrib/zstd"]
 	path = contrib/zstd
@@ -157,6 +157,9 @@
 [submodule "contrib/openldap"]
 	path = contrib/openldap
 	url = https://github.com/openldap/openldap.git
+[submodule "contrib/AMQP-CPP"]
+	path = contrib/AMQP-CPP
+	url = https://github.com/CopernicaMarketingSoftware/AMQP-CPP.git
 [submodule "contrib/cassandra"]
 	path = contrib/cassandra
 	url = https://github.com/ClickHouse-Extras/cpp-driver.git
@@ -168,3 +171,6 @@
 [submodule "contrib/fmtlib"]
 	path = contrib/fmtlib
 	url = https://github.com/fmtlib/fmt.git
+[submodule "contrib/sentry-native"]
+	path = contrib/sentry-native
+	url = https://github.com/getsentry/sentry-native.git
diff --git a/CHANGELOG.md b/CHANGELOG.md
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -289,8 +289,9 @@ set (CMAKE_POSTFIX_VARIABLE "CMAKE_${CMAKE_BUILD_TYPE_UC}_POSTFIX")
 
 if (MAKE_STATIC_LIBRARIES)
     set (CMAKE_POSITION_INDEPENDENT_CODE OFF)
-    if (OS_LINUX)
+    if (OS_LINUX AND NOT ARCH_ARM)
         # Slightly more efficient code can be generated
+        # It's disabled for ARM because otherwise ClickHouse cannot run on Android.
         set (CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-pie")
         set (CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO} -fno-pie")
         set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-no-pie")
@@ -341,6 +342,7 @@ include (cmake/find/sparsehash.cmake)
 include (cmake/find/re2.cmake)
 include (cmake/find/libgsasl.cmake)
 include (cmake/find/rdkafka.cmake)
+include (cmake/find/amqpcpp.cmake)
 include (cmake/find/capnp.cmake)
 include (cmake/find/llvm.cmake)
 include (cmake/find/opencl.cmake)
@@ -361,6 +363,7 @@ include (cmake/find/orc.cmake)
 include (cmake/find/avro.cmake)
 include (cmake/find/msgpack.cmake)
 include (cmake/find/cassandra.cmake)
+include (cmake/find/sentry.cmake)
 
 find_contrib_lib(cityhash)
 find_contrib_lib(farmhash)
@@ -418,3 +421,5 @@ add_subdirectory (tests)
 add_subdirectory (utils)
 
 include (cmake/print_include_directories.cmake)
+
+include (cmake/sanitize_target_link_libraries.cmake)
diff --git a/README.md b/README.md
@@ -14,7 +14,7 @@ ClickHouse is an open-source column-oriented database management system that all
 * [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any.
 * You can also [fill this form](https://clickhouse.tech/#meet) to meet Yandex ClickHouse team in person.
 
-## Upcoming Events
+## Upcoming Events		
 
-* [ClickHouse Workshop in Novosibirsk](https://2020.codefest.ru/lecture/1628) on TBD date.
-* [Yandex C++ Open-Source Sprints in Moscow](https://events.yandex.ru/events/otkrytyj-kod-v-yandek-28-03-2020) on TBD date.
+* [ClickHouse for genetic data (in Russian)](https://cloud.yandex.ru/events/152) on July 14, 2020.
+* [ClickHouse virtual office hours](https://www.meetup.com/San-Francisco-Bay-Area-ClickHouse-Meetup/events/271522978/) on July 15, 2020.
diff --git a/SECURITY.md b/SECURITY.md
@@ -11,7 +11,10 @@ currently being supported with security updates:
 | 18.x   | :x:                |
 | 19.x   | :x:                |
 | 19.14   | :white_check_mark: |
-| 20.x   | :white_check_mark: |
+| 20.1   | :x: |
+| 20.3   | :white_check_mark: |
+| 20.4   | :white_check_mark: |
+| 20.5   | :white_check_mark: |
 
 ## Reporting a Vulnerability
 

diff --git a/base/common/CMakeLists.txt b/base/common/CMakeLists.txt
@@ -77,10 +77,8 @@ target_link_libraries (common
         Poco::Util
         Poco::Foundation
         replxx
-        fmt
-
-    PRIVATE
         cctz
+        fmt
 )
 
 if (ENABLE_TESTS)

diff --git a/base/common/DateLUT.cpp b/base/common/DateLUT.cpp
@@ -75,7 +75,7 @@ std::string determineDefaultTimeZone()
 
     try
     {
-        tz_database_path = fs::canonical(tz_database_path);
+        tz_database_path = fs::weakly_canonical(tz_database_path);
 
         /// The tzdata file exists. If it is inside the tz_database_dir,
         /// then the relative path is the time zone id.
@@ -91,7 +91,7 @@ std::string determineDefaultTimeZone()
             if (!tz_file_path.is_absolute())
                 tz_file_path = tz_database_path / tz_file_path;
 
-            tz_file_path = fs::canonical(tz_file_path);
+            tz_file_path = fs::weakly_canonical(tz_file_path);
 
             fs::path relative_path = tz_file_path.lexically_relative(tz_database_path);
             if (!relative_path.empty() && *relative_path.begin() != ".." && *relative_path.begin() != ".")

diff --git a/base/common/LineReader.h b/base/common/LineReader.h
@@ -48,7 +48,7 @@ class LineReader
     };
 
     const String history_file_path;
-    static constexpr char word_break_characters[] = " \t\n\r\"\\'`@$><=;|&{(.";
+    static constexpr char word_break_characters[] = " \t\v\f\a\b\r\n`~!@#$%^&*()-=+[{]}\\|;:'\",<.>/?";
 
     String input;
 

diff --git a/base/common/ya.make b/base/common/ya.make
@@ -2,7 +2,7 @@ LIBRARY()
 
 ADDINCL(
     GLOBAL clickhouse/base
-    contrib/libs/cctz/include
+    GLOBAL contrib/libs/cctz/include
 )
 
 CFLAGS (GLOBAL -DARCADIA_BUILD)