Merge branch 'master' into enable_atomic_database_by_default

.gitmodules

@@ -186,4 +186,3 @@
 [submodule "contrib/cyrus-sasl"]
 	path = contrib/cyrus-sasl
 	url = https://github.com/cyrusimap/cyrus-sasl
-	branch = cyrus-sasl-2.1

README.md

@@ -17,5 +17,5 @@ ClickHouse is an open-source column-oriented database management system that all
 
 ## Upcoming Events
 
-* [eBay migrating from Druid](https://us02web.zoom.us/webinar/register/tZMkfu6rpjItHtaQ1DXcgPWcSOnmM73HLGKL) on September 23, 2020.
 * [ClickHouse for Edge Analytics](https://ones2020.sched.com/event/bWPs) on September 29, 2020.
+* [ClickHouse online meetup (in Russian)](https://clck.ru/R2zB9) on October 1, 2020.

cmake/find/rdkafka.cmake

@@ -14,10 +14,10 @@ if (NOT ENABLE_RDKAFKA)
     return()
 endif()
 
-if (NOT ARCH_ARM)
+if (NOT ARCH_ARM AND USE_LIBGSASL)
     option (USE_INTERNAL_RDKAFKA_LIBRARY "Set to FALSE to use system librdkafka instead of the bundled" ${NOT_UNBUNDLED})
 elseif(USE_INTERNAL_RDKAFKA_LIBRARY)
-    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ARCH_ARM=${ARCH_ARM}")
+    message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use internal librdkafka with ARCH_ARM=${ARCH_ARM} AND USE_LIBGSASL=${USE_LIBGSASL}")
 endif ()
 
 if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/cppkafka/CMakeLists.txt")

contrib/protobuf-cmake/CMakeLists.txt

@@ -11,3 +11,7 @@ else ()
 endif ()
 
 add_subdirectory("${protobuf_SOURCE_DIR}/cmake" "${protobuf_BINARY_DIR}")
+
+# We don't want to stop compilation on warnings in protobuf's headers.
+# The following line overrides the value assigned by the command target_include_directories() in libprotobuf.cmake 
+set_property(TARGET libprotobuf PROPERTY INTERFACE_SYSTEM_INCLUDE_DIRECTORIES ${protobuf_SOURCE_DIR}/src)

debian/rules

@@ -36,8 +36,8 @@ endif
 
 CMAKE_FLAGS += -DENABLE_UTILS=0
 
-DEB_CC ?= $(shell which gcc-9 gcc-8 gcc | head -n1)
-DEB_CXX ?= $(shell which g++-9 g++-8 g++ | head -n1)
+DEB_CC ?= $(shell which gcc-10 gcc-9 gcc | head -n1)
+DEB_CXX ?= $(shell which g++-10 g++-9 g++ | head -n1)
 
 ifdef DEB_CXX
     DEB_BUILD_GNU_TYPE := $(shell dpkg-architecture -qDEB_BUILD_GNU_TYPE)

docker/images.json

@@ -133,10 +133,6 @@
         "name": "yandex/clickhouse-postgresql-java-client",
         "dependent": []
     },
-    "docker/test/integration/kerberos_kdc": {
-        "name": "yandex/clickhouse-kerberos-kdc",
-        "dependent": []
-    },
     "docker/test/base": {
          "name": "yandex/clickhouse-test-base",
          "dependent": [

docker/test/fasttest/run.sh

@@ -97,7 +97,7 @@ ccache --zero-stats ||:
 mkdir build
 cd build
 cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_CXX_COMPILER=clang++-10 -DCMAKE_C_COMPILER=clang-10 "${CMAKE_LIBS_CONFIG[@]}" "${FASTTEST_CMAKE_FLAGS[@]}" | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/cmake_log.txt
-ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/build_log.txt
+time ninja clickhouse-bundle | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/build_log.txt
 ninja install | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/install_log.txt
 
 
@@ -111,36 +111,10 @@ ln -s /test_output /var/log/clickhouse-server
 cp "$CLICKHOUSE_DIR/programs/server/config.xml" /etc/clickhouse-server/
 cp "$CLICKHOUSE_DIR/programs/server/users.xml" /etc/clickhouse-server/
 
-mkdir -p /etc/clickhouse-server/dict_examples
-ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/dict_examples/
-ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/dict_examples/
-ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/dict_examples/
-ln -s /usr/share/clickhouse-test/config/zookeeper.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/listen.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/part_log.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/text_log.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/metric_log.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/custom_settings_prefixes.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/log_queries.xml /etc/clickhouse-server/users.d/
-ln -s /usr/share/clickhouse-test/config/readonly.xml /etc/clickhouse-server/users.d/
-ln -s /usr/share/clickhouse-test/config/access_management.xml /etc/clickhouse-server/users.d/
-ln -s /usr/share/clickhouse-test/config/ints_dictionary.xml /etc/clickhouse-server/
-ln -s /usr/share/clickhouse-test/config/strings_dictionary.xml /etc/clickhouse-server/
-ln -s /usr/share/clickhouse-test/config/decimals_dictionary.xml /etc/clickhouse-server/
-ln -s /usr/share/clickhouse-test/config/executable_dictionary.xml /etc/clickhouse-server/
-ln -s /usr/share/clickhouse-test/config/macros.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/disks.xml /etc/clickhouse-server/config.d/
-#ln -s /usr/share/clickhouse-test/config/secure_ports.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/clusters.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/graphite.xml /etc/clickhouse-server/config.d/
-ln -s /usr/share/clickhouse-test/config/server.key /etc/clickhouse-server/
-ln -s /usr/share/clickhouse-test/config/server.crt /etc/clickhouse-server/
-ln -s /usr/share/clickhouse-test/config/dhparam.pem /etc/clickhouse-server/
-ln -s /usr/share/clickhouse-test/config/database_atomic_configd.xml /etc/clickhouse-server/config.d/
-ln -sf /usr/share/clickhouse-test/config/client_config.xml /etc/clickhouse-client/config.xml
-
-# Keep original query_masking_rules.xml
-ln -s --backup=simple --suffix=_original.xml /usr/share/clickhouse-test/config/query_masking_rules.xml /etc/clickhouse-server/config.d/
+# install tests config
+$CLICKHOUSE_DIR/tests/config/install.sh
+# doesn't support SSL
+rm -f /etc/clickhouse-server/config.d/secure_ports.xml
 
 # Kill the server in case we are running locally and not in docker
 kill_clickhouse
@@ -217,7 +191,7 @@ TESTS_TO_SKIP=(
     01460_DistributedFilesToInsert
 )
 
-clickhouse-test -j 4 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt
+time clickhouse-test -j 8 --no-long --testname --shard --zookeeper --skip "${TESTS_TO_SKIP[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_output/test_log.txt
 
 
 # substr is to remove semicolon after test name
@@ -235,7 +209,7 @@ then
     kill_clickhouse
 
     # Clean the data so that there is no interference from the previous test run.
-    rm -rvf /var/lib/clickhouse ||:
+    rm -rf /var/lib/clickhouse ||:
     mkdir /var/lib/clickhouse
 
     clickhouse-server --config /etc/clickhouse-server/config.xml --daemon

docker/test/fuzzer/run-fuzzer.sh

@@ -48,7 +48,7 @@ function configure
     cp -av "$repo_dir"/programs/server/config* db
     cp -av "$repo_dir"/programs/server/user* db
     # TODO figure out which ones are needed
-    cp -av "$repo_dir"/tests/config/listen.xml db/config.d
+    cp -av "$repo_dir"/tests/config/config.d/listen.xml db/config.d
     cp -av "$script_dir"/query-fuzzer-tweaks-users.xml db/users.d
 }
 

docker/test/integration/base/Dockerfile

@@ -16,8 +16,7 @@ RUN apt-get update \
         odbc-postgresql \
         sqlite3 \
         curl \
-        tar \
-        krb5-user
+        tar
 RUN rm -rf \
         /var/lib/apt/lists/* \
         /var/cache/debconf \

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -27,7 +27,6 @@ export DOCKER_MYSQL_JAVA_CLIENT_TAG=${DOCKER_MYSQL_JAVA_CLIENT_TAG:=latest}
 export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest}
 export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest}
 export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest}
-export DOCKER_KERBEROS_KDC_TAG=${DOCKER_KERBEROS_KDC_TAG:=latest}
 
 cd /ClickHouse/tests/integration
 exec "$@"

docker/test/performance-comparison/compare.sh

@@ -726,8 +726,8 @@ create view shortness
 create table inconsistent_short_marking_report
     engine File(TSV, 'report/unexpected-query-duration.tsv')
     as select
-        multiIf(marked_short and time > 0.1, '"short" queries must run faster than 0.02 s',
-                not marked_short and time < 0.02, '"normal" queries must run longer than 0.1 s',
+        multiIf(marked_short and time > 0.1, '\"short\" queries must run faster than 0.02 s',
+                not marked_short and time < 0.02, '\"normal\" queries must run longer than 0.1 s',
                 '') problem,
         marked_short, time,
         test, query_index, query_display_name
@@ -1065,7 +1065,7 @@ case "$stage" in
     # to collect the logs. Prefer not to restart, because addresses might change
     # and we won't be able to process trace_log data. Start in a subshell, so that
     # it doesn't interfere with the watchdog through `wait`.
-    ( get_profiles || restart && get_profiles ) ||:
+    ( get_profiles || { restart && get_profiles ; } ) ||:
 
     # Kill the whole process group, because somehow when the subshell is killed,
     # the sleep inside remains alive and orphaned.

docker/test/performance-comparison/perf.py

@@ -15,6 +15,7 @@
 import time
 import traceback
 import xml.etree.ElementTree as et
+from threading import Thread
 from scipy import stats
 
 def tsv_escape(s):
@@ -23,10 +24,11 @@ def tsv_escape(s):
 parser = argparse.ArgumentParser(description='Run performance test.')
 # Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set.
 parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file')
-parser.add_argument('--host', nargs='*', default=['localhost'], help="Server hostname(s). Corresponds to '--port' options.")
-parser.add_argument('--port', nargs='*', default=[9000], help="Server port(s). Corresponds to '--host' options.")
+parser.add_argument('--host', nargs='*', default=['localhost'], help="Space-separated list of server hostname(s). Corresponds to '--port' options.")
+parser.add_argument('--port', nargs='*', default=[9000], help="Space-separated list of server port(s). Corresponds to '--host' options.")
 parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.')
 parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.')
+parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.')
 parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.')
 parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.')
 parser.add_argument('--print-settings', action='store_true', help='Print test settings and exit.')
@@ -157,8 +159,11 @@ def substitute_parameters(query_templates, other_templates = []):
             print(f'skipped\t{tsv_escape(skipped_message)}')
             sys.exit(0)
 
-# Run create queries
-create_query_templates = [q.text for q in root.findall('create_query')]
+# Run create and fill queries. We will run them simultaneously for both servers,
+# to save time.
+# The weird search is to keep the relative order of elements, which matters, and
+# etree doesn't support the appropriate xpath query.
+create_query_templates = [q.text for q in root.findall('./*') if q.tag in ('create_query', 'fill_query')]
 create_queries = substitute_parameters(create_query_templates)
 
 # Disallow temporary tables, because the clickhouse_driver reconnects on errors,
@@ -170,23 +175,34 @@ def substitute_parameters(query_templates, other_templates = []):
             file = sys.stderr)
         sys.exit(1)
 
-for conn_index, c in enumerate(all_connections):
-    for q in create_queries:
-        c.execute(q)
-        print(f'create\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
+def do_create(connection, index, queries):
+    for q in queries:
+        connection.execute(q)
+        print(f'create\t{index}\t{connection.last_query.elapsed}\t{tsv_escape(q)}')
 
-# Run fill queries
-fill_query_templates = [q.text for q in root.findall('fill_query')]
-fill_queries = substitute_parameters(fill_query_templates)
-for conn_index, c in enumerate(all_connections):
-    for q in fill_queries:
-        c.execute(q)
-        print(f'fill\t{conn_index}\t{c.last_query.elapsed}\t{tsv_escape(q)}')
+threads = [Thread(target = do_create, args = (connection, index, create_queries))
+                for index, connection in enumerate(all_connections)]
+
+for t in threads:
+    t.start()
+
+for t in threads:
+    t.join()
+
+queries_to_run = range(0, len(test_queries))
+
+if args.max_queries:
+    # If specified, test a limited number of queries chosen at random.
+    queries_to_run = random.sample(range(0, len(test_queries)), min(len(test_queries), args.max_queries))
+
+if args.queries_to_run:
+    # Run the specified queries, with some sanity check.
+    for i in args.queries_to_run:
+        if i < 0 or i >= len(test_queries):
+            print(f'There is no query no. "{i}" in this test, only [{0}-{len(test_queries) - 1}] are present')
+            exit(1)
 
-# Run the queries in randomized order, but preserve their indexes as specified
-# in the test XML. To avoid using too much time, limit the number of queries
-# we run per test.
-queries_to_run = random.sample(range(0, len(test_queries)), min(len(test_queries), args.max_queries or len(test_queries)))
+    queries_to_run = args.queries_to_run
 
 # Run test queries.
 for query_index in queries_to_run: