Update tests' docker containers

docker/test/fasttest/Dockerfile

@@ -9,6 +9,7 @@ RUN apt-get update \
         expect \
         file \
         lsof \
+        odbcinst \
         psmisc \
         python3 \
         python3-lxml \

docker/test/fasttest/run.sh

@@ -80,7 +80,7 @@ function start_server
 
 function clone_root
 {
-    git config --global --add safe.directory "$FASTTEST_SOURCE"
+    [ "$UID" -eq 0 ] && git config --global --add safe.directory "$FASTTEST_SOURCE"
     git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt"
 
     (
@@ -151,7 +151,7 @@ function clone_submodules
         )
 
         git submodule sync
-        git submodule update --jobs=16 --depth 1 --init "${SUBMODULES_TO_UPDATE[@]}"
+        git submodule update --jobs=16 --depth 1 --single-branch --init "${SUBMODULES_TO_UPDATE[@]}"
         git submodule foreach git reset --hard
         git submodule foreach git checkout @ -f
         git submodule foreach git clean -xfd
@@ -202,10 +202,11 @@ function build
           | ts '%Y-%m-%d %H:%M:%S' \
           | tee "$FASTTEST_OUTPUT/test_result.txt"
         if [ "$COPY_CLICKHOUSE_BINARY_TO_OUTPUT" -eq "1" ]; then
-            cp programs/clickhouse "$FASTTEST_OUTPUT/clickhouse"
+            mkdir -p "$FASTTEST_OUTPUT/binaries/"
+            cp programs/clickhouse "$FASTTEST_OUTPUT/binaries/clickhouse"
 
-            strip programs/clickhouse -o "$FASTTEST_OUTPUT/clickhouse-stripped"
-            zstd --threads=0 "$FASTTEST_OUTPUT/clickhouse-stripped"
+            strip programs/clickhouse -o programs/clickhouse-stripped
+            zstd --threads=0 programs/clickhouse-stripped -o "$FASTTEST_OUTPUT/binaries/clickhouse-stripped.zst"
         fi
         ccache_status
         ccache --evict-older-than 1d ||:

docker/test/integration/base/Dockerfile

@@ -46,12 +46,13 @@ RUN arch=${TARGETARCH:-amd64} \
       arm64) rarch=aarch64 ;; \
     esac \
   && cd /tmp \
-  && curl -o mysql-odbc.rpm "https://cdn.mysql.com/archives/mysql-connector-odbc-8.0/mysql-connector-odbc-8.0.27-1.el8.${rarch}.rpm" \
+  && curl -o mysql-odbc.rpm "https://cdn.mysql.com/archives/mysql-connector-odbc-8.0/mysql-connector-odbc-8.0.32-1.el9.${rarch}.rpm" \
   && rpm2archive mysql-odbc.rpm \
   && tar xf mysql-odbc.rpm.tgz -C / ./usr/lib64/ \
-  && LINK_DIR=$(dpkg -L libodbc1 | rg '^/usr/lib/.*-linux-gnu/odbc$') \
-  && ln -s /usr/lib64/libmyodbc8a.so "$LINK_DIR" \
-  && ln -s /usr/lib64/libmyodbc8a.so "$LINK_DIR"/libmyodbc.so
+  && rm mysql-odbc.rpm mysql-odbc.rpm.tgz \
+  && ODBC_DIR=$(dpkg -L odbc-postgresql | rg '^/usr/lib/.*-linux-gnu/odbc$') \
+  && ln -s /usr/lib64/libmyodbc8a.so "$ODBC_DIR" \
+  && ln -s /usr/lib64/libmyodbc8a.so "$ODBC_DIR"/libmyodbc.so
 
 # Unfortunately this is required for a single test for conversion data from zookeeper to clickhouse-keeper.
 # ZooKeeper is not started by default, but consumes some space in containers.

docker/test/integration/helper_container/Dockerfile

@@ -2,4 +2,7 @@
 # Helper docker container to run iptables without sudo
 
 FROM alpine
-RUN apk add -U iproute2
+RUN apk add --no-cache -U iproute2 \
+  && for bin in iptables iptables-restore iptables-save; \
+    do ln -sf xtables-nft-multi "/sbin/$bin"; \
+  done

docker/test/integration/mysql_php_client/Dockerfile

@@ -1,7 +1,7 @@
 # docker build -t clickhouse/mysql-php-client .
 # MySQL PHP client docker container
 
-FROM php:8.0.18-cli
+FROM php:8-cli-alpine
 
 COPY ./client.crt client.crt
 COPY ./client.key client.key

docker/test/integration/runner/Dockerfile

@@ -1,5 +1,5 @@
 # docker build -t clickhouse/integration-tests-runner .
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"
@@ -56,17 +56,19 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \
         /var/lib/apt/lists/* \
         /var/cache/debconf \
         /tmp/* \
-    && apt-get clean
+    && apt-get clean \
+    && dockerd --version; docker --version
 
-RUN dockerd --version; docker --version
 
 RUN python3 -m pip install --no-cache-dir \
     PyMySQL \
-    aerospike==4.0.0 \
-    avro==1.10.2 \
+    aerospike==11.1.0 \
     asyncio \
+    avro==1.10.2 \
+    azure-storage-blob \
     cassandra-driver \
-    confluent-kafka==1.5.0 \
+    confluent-kafka==1.9.2 \
+    delta-spark==2.3.0 \
     dict2xml \
     dicttoxml \
     docker \
@@ -76,47 +78,51 @@ RUN python3 -m pip install --no-cache-dir \
     kafka-python \
     kazoo \
     lz4 \
+    meilisearch==0.18.3 \
     minio \
     nats-py \
     protobuf \
-    psycopg2-binary==2.8.6 \
+    psycopg2-binary==2.9.6 \
+    pyhdfs \
     pymongo==3.11.0 \
+    pyspark==3.3.2 \
     pytest \
     pytest-order==1.0.0 \
-    pytest-timeout \
     pytest-random \
-    pytest-xdist \
     pytest-repeat \
+    pytest-timeout \
+    pytest-xdist \
     pytz \
     redis \
-    tzlocal==2.1 \
-    urllib3 \
     requests-kerberos \
-    pyspark==3.3.2 \
-    delta-spark==2.2.0 \
-    pyhdfs \
-    azure-storage-blob \
-    meilisearch==0.18.3
-
-COPY modprobe.sh /usr/local/bin/modprobe
-COPY dockerd-entrypoint.sh /usr/local/bin/
-COPY compose/ /compose/
-COPY misc/ /misc/
+    tzlocal==2.1 \
+    urllib3
 
+# Hudi supports only spark 3.3.*, not 3.4
 RUN curl -fsSL -O https://dlcdn.apache.org/spark/spark-3.3.2/spark-3.3.2-bin-hadoop3.tgz \
     && tar xzvf spark-3.3.2-bin-hadoop3.tgz -C / \
     && rm spark-3.3.2-bin-hadoop3.tgz
 
 # download spark and packages
 # if you change packages, don't forget to update them in tests/integration/helpers/cluster.py
-RUN echo ":quit" | /spark-3.3.2-bin-hadoop3/bin/spark-shell --packages "org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0,io.delta:delta-core_2.12:2.2.0,org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.1.0" > /dev/null
+RUN packages="org.apache.hudi:hudi-spark3.3-bundle_2.12:0.13.0,\
+io.delta:delta-core_2.12:2.3.0,\
+org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.1.0" \
+    && /spark-3.3.2-bin-hadoop3/bin/spark-shell --packages "$packages" > /dev/null \
+    && find /root/.ivy2/ -name '*.jar' -exec ln -sf {} /spark-3.3.2-bin-hadoop3/jars/ \;
 
 RUN set -x \
   && addgroup --system dockremap \
-    && adduser --system dockremap \
+  && adduser --system dockremap \
   && adduser dockremap dockremap \
   && echo 'dockremap:165536:65536' >> /etc/subuid \
-    && echo 'dockremap:165536:65536' >> /etc/subgid
+  && echo 'dockremap:165536:65536' >> /etc/subgid
+
+COPY modprobe.sh /usr/local/bin/modprobe
+COPY dockerd-entrypoint.sh /usr/local/bin/
+COPY compose/ /compose/
+COPY misc/ /misc/
+
 
 # Same options as in test/base/Dockerfile
 # (in case you need to override them in tests)

docker/test/integration/runner/dockerd-entrypoint.sh

@@ -12,6 +12,17 @@ echo '{
     "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"]
 }' | dd of=/etc/docker/daemon.json 2>/dev/null
 
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+    # move the processes from the root group to the /init group,
+    # otherwise writing subtree_control fails with EBUSY.
+    # An error during moving non-existent process (i.e., "cat") is ignored.
+    mkdir -p /sys/fs/cgroup/init
+    xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+    # enable controllers
+    sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers \
+        > /sys/fs/cgroup/cgroup.subtree_control
+fi
+
 # In case of test hung it is convenient to use pytest --pdb to debug it,
 # and on hung you can simply press Ctrl-C and it will spawn a python pdb,
 # but on SIGINT dockerd will exit, so ignore it to preserve the daemon.
@@ -52,6 +63,8 @@ export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=/clickhouse-config
 export CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH=/clickhouse-odbc-bridge
 export CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH=/clickhouse-library-bridge
 
+export DOCKER_BASE_TAG=${DOCKER_BASE_TAG:=latest}
+export DOCKER_HELPER_TAG=${DOCKER_HELPER_TAG:=latest}
 export DOCKER_MYSQL_GOLANG_CLIENT_TAG=${DOCKER_MYSQL_GOLANG_CLIENT_TAG:=latest}
 export DOCKER_DOTNET_CLIENT_TAG=${DOCKER_DOTNET_CLIENT_TAG:=latest}
 export DOCKER_MYSQL_JAVA_CLIENT_TAG=${DOCKER_MYSQL_JAVA_CLIENT_TAG:=latest}

docker/test/stateful/Dockerfile

@@ -16,8 +16,9 @@ COPY s3downloader /s3downloader
 ENV S3_URL="https://clickhouse-datasets.s3.amazonaws.com"
 ENV DATASETS="hits visits"
 
-RUN npm install -g azurite
-RUN npm install tslib
+# The following is already done in clickhouse/stateless-test
+# RUN npm install -g azurite
+# RUN npm install tslib
 
 COPY run.sh /
 CMD ["/bin/bash", "/run.sh"]

docker/test/stateless/Dockerfile

@@ -20,6 +20,7 @@ RUN apt-get update -y \
             netcat-openbsd \
             nodejs \
             npm \
+            odbcinst \
             openjdk-11-jre-headless \
             openssl \
             postgresql-client \
@@ -71,16 +72,16 @@ RUN arch=${TARGETARCH:-amd64} \
     && chmod +x ./mc ./minio
 
 
-RUN wget 'https://dlcdn.apache.org/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
+RUN wget --no-verbose 'https://dlcdn.apache.org/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
     && tar -xvf hadoop-3.3.1.tar.gz \
     && rm -rf hadoop-3.3.1.tar.gz
 
 ENV MINIO_ROOT_USER="clickhouse"
 ENV MINIO_ROOT_PASSWORD="clickhouse"
 ENV EXPORT_S3_STORAGE_POLICIES=1
 
-RUN npm install -g azurite
-RUN npm install tslib
+RUN npm install -g azurite \
+    && npm install -g tslib
 
 COPY run.sh /
 COPY setup_minio.sh /

docker/test/util/Dockerfile

@@ -1,5 +1,5 @@
 # docker build -t clickhouse/test-util .
-FROM ubuntu:20.04
+FROM ubuntu:22.04
 
 # ARG for quick switch to a given ubuntu mirror
 ARG apt_archive="http://archive.ubuntu.com"

src/Common/getNumberOfPhysicalCPUCores.cpp

@@ -1,4 +1,5 @@
 #include "getNumberOfPhysicalCPUCores.h"
+#include <filesystem>
 
 #include "config.h"
 #if defined(OS_LINUX)
@@ -7,6 +8,8 @@
 #endif
 
 #include <boost/algorithm/string/trim.hpp>
+#include <boost/algorithm/string/split.hpp>
+#include <base/range.h>
 
 #include <thread>
 #include <set>
@@ -15,7 +18,7 @@ namespace
 {
 
 #if defined(OS_LINUX)
-int32_t readFrom(const char * filename, int default_value)
+int32_t readFrom(const std::filesystem::path & filename, int default_value)
 {
     std::ifstream infile(filename);
     if (!infile.is_open())
@@ -31,10 +34,87 @@ int32_t readFrom(const char * filename, int default_value)
 uint32_t getCGroupLimitedCPUCores(unsigned default_cpu_count)
 {
     uint32_t quota_count = default_cpu_count;
+    std::filesystem::path prefix = "/sys/fs/cgroup";
+    /// cgroupsv2
+    std::ifstream contr_file(prefix / "cgroup.controllers");
+    if (contr_file.is_open())
+    {
+        /// First, we identify the cgroup the process belongs
+        std::ifstream cgroup_name_file("/proc/self/cgroup");
+        if (!cgroup_name_file.is_open())
+            return default_cpu_count;
+
+        // cgroup_name_file always starts with '0::/' for v2
+        cgroup_name_file.ignore(4);
+        std::string cgroup_name;
+        cgroup_name_file >> cgroup_name;
+
+        std::filesystem::path current_cgroup;
+        if (cgroup_name.empty())
+            current_cgroup = prefix;
+        else
+            current_cgroup = prefix / cgroup_name;
+
+        // Looking for cpu.max in directories from the current cgroup to the top level
+        // It does not stop on the first time since the child could have a greater value than parent
+        while (current_cgroup != prefix.parent_path())
+        {
+            std::ifstream cpu_max_file(current_cgroup / "cpu.max");
+            current_cgroup = current_cgroup.parent_path();
+            if (cpu_max_file.is_open())
+            {
+                std::string cpu_limit_str;
+                float cpu_period;
+                cpu_max_file >> cpu_limit_str >> cpu_period;
+                if (cpu_limit_str != "max" && cpu_period != 0)
+                {
+                    float cpu_limit = std::stof(cpu_limit_str);
+                    quota_count = std::min(static_cast<uint32_t>(ceil(cpu_limit / cpu_period)), quota_count);
+                }
+            }
+        }
+        current_cgroup = prefix / cgroup_name;
+        // Looking for cpuset.cpus.effective in directories from the current cgroup to the top level
+        while (current_cgroup != prefix.parent_path())
+        {
+            std::ifstream cpuset_cpus_file(current_cgroup / "cpuset.cpus.effective");
+            current_cgroup = current_cgroup.parent_path();
+            if (cpuset_cpus_file.is_open())
+            {
+                // The line in the file is "0,2-4,6,9-14" cpu numbers
+                // It's always grouped and ordered
+                std::vector<std::string> cpu_ranges;
+                std::string cpuset_line;
+                cpuset_cpus_file >> cpuset_line;
+                if (cpuset_line.empty())
+                    continue;
+                boost::split(cpu_ranges, cpuset_line, boost::is_any_of(","));
+                uint32_t cpus_count = 0;
+                for (const std::string& cpu_number_or_range : cpu_ranges)
+                {
+                    std::vector<std::string> cpu_range;
+                    boost::split(cpu_range, cpu_number_or_range, boost::is_any_of("-"));
+
+                    if (cpu_range.size() == 2)
+                    {
+                        int start = std::stoi(cpu_range[0]);
+                        int end = std::stoi(cpu_range[1]);
+                        cpus_count += (end - start) + 1;
+                    }
+                    else
+                        cpus_count++;
+                }
+                quota_count = std::min(cpus_count, quota_count);
+                break;
+            }
+        }
+        return quota_count;
+    }
+    /// cgroupsv1
     /// Return the number of milliseconds per period process is guaranteed to run.
     /// -1 for no quota
-    int cgroup_quota = readFrom("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", -1);
-    int cgroup_period = readFrom("/sys/fs/cgroup/cpu/cpu.cfs_period_us", -1);
+    int cgroup_quota = readFrom(prefix / "cpu/cpu.cfs_quota_us", -1);
+    int cgroup_period = readFrom(prefix / "cpu/cpu.cfs_period_us", -1);
     if (cgroup_quota > -1 && cgroup_period > 0)
         quota_count = static_cast<uint32_t>(ceil(static_cast<float>(cgroup_quota) / static_cast<float>(cgroup_period)));