From 856815b183f24e9e4fce8da9bfefa508c2d25a00 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 17 Jan 2023 15:43:49 +0400 Subject: [PATCH 01/11] Merge pull request #218 from zvonand/22.8-fips-go1.19-boringssl Tuned CI/CD for FIPs build --- docker/packager/binary/Dockerfile | 40 ++++++++++++++++++++++++++++-- docker/packager/binary/build.sh | 22 ++++++++++++++++ docker/packager/binary/daemon.json | 8 ++++++ docker/packager/packager | 13 +++++++++- tests/ci/build_check.py | 2 +- 5 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 docker/packager/binary/daemon.json diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index adc0b676144c..9b0b9be96e90 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -1,9 +1,29 @@ + # docker build -t altinityinfra/binary-builder . ARG FROM_TAG=latest FROM altinityinfra/test-util:$FROM_TAG as cctools # The cctools are built always from the altinityinfra/test-util:latest and cached inline # Theoretically, it should improve rebuild speed significantly +RUN apt update && apt install cmake --yes + + +# Install Docker +ENV DOCKER_CHANNEL stable +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +RUN add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + docker-ce \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/debconf \ + /tmp/* \ + && apt-get clean + +RUN dockerd --version; docker --version + ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -36,6 +56,22 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ && cd ../.. \ && rm -rf cctools-port +# Install Docker +ENV DOCKER_CHANNEL stable +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +RUN add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + docker-ce \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/debconf \ + /tmp/* \ + && apt-get clean + +RUN dockerd --version; docker --version + # !!!!!!!!!!! # END COMPILE # !!!!!!!!!!! @@ -105,9 +141,9 @@ RUN curl -Lo /usr/bin/clang-tidy-cache \ && chmod +x /usr/bin/clang-tidy-cache COPY --from=cctools /cctools /cctools - RUN mkdir /workdir && chmod 777 /workdir WORKDIR /workdir - COPY build.sh / +COPY daemon.json /etc/docker/ + CMD ["bash", "-c", "/build.sh 2>&1"] diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 23bdad162522..242fb2267dbb 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -1,4 +1,26 @@ #!/usr/bin/env bash + +set -e + +# In case of test hung it is convenient to use pytest --pdb to debug it, +# and on hung you can simply press Ctrl-C and it will spawn a python pdb, +# but on SIGINT dockerd will exit, so ignore it to preserve the daemon. +trap '' INT +dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 & + +set +e +reties=0 +while true; do + docker info &>/dev/null && break + reties=$((reties+1)) + if [[ $reties -ge 100 ]]; then # 10 sec max + echo "Can't start docker daemon, timeout exceeded." >&2 + exit 1; + fi + sleep 0.1 +done +set -e + set -x -e exec &> >(ts) diff --git a/docker/packager/binary/daemon.json b/docker/packager/binary/daemon.json new file mode 100644 index 000000000000..d309c3a52c8a --- /dev/null +++ b/docker/packager/binary/daemon.json @@ -0,0 +1,8 @@ +{ + "ipv6": true, + "fixed-cidr-v6": "fd00::/8", + "ip-forward": true, + "log-level": "debug", + "insecure-registries" : ["dockerhub-proxy.dockerhub-proxy-zone:5000"], + "registry-mirrors" : ["http://dockerhub-proxy.dockerhub-proxy-zone:5000"] +} \ No newline at end of file diff --git a/docker/packager/packager b/docker/packager/packager index 2228571d0e7c..b04e28fd1d41 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -101,8 +101,10 @@ def run_docker_image_with_env( if ccache_dir is None: ccache_mount = "" + # Added v /var/run/docker.sock:/var/run/docker.sock, required for docker-in-docker to build BoringSSL in FIPS mode. cmd = ( - f"docker run --network=host --user={user} --rm {ccache_mount}" + f"docker run --network=host --user={user} --rm {ccache_mount} " + f" -v /var/run/docker.sock:/var/run/docker.sock " f" --volume={output_dir}:/output --volume={ch_root}:/build {env_part} " f"{interactive} {image_name}" ) @@ -131,6 +133,7 @@ def parse_env_variables( additional_pkgs: bool, with_coverage: bool, with_binaries: str, + fips, ): DARWIN_SUFFIX = "-darwin" DARWIN_ARM_SUFFIX = "-darwin-aarch64" @@ -212,6 +215,10 @@ def parse_env_variables( cc = compiler result.append("DEB_ARCH=amd64") + if fips: + cmake_flags.append("-DLINKER_NAME=/usr/lib/llvm-14/bin/ld.lld") + cmake_flags.append("-DFIPS_CLICKHOUSE=ON") + cxx = cc.replace("gcc", "g++").replace("clang", "clang++") if package_type == "deb": @@ -422,6 +429,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--as-root", action="store_true", help="if the container should run as root" ) + parser.add_argument( + "--fips", action="store_true", help="enbale FIPS build", default=False + ) args = parser.parse_args() @@ -477,6 +487,7 @@ def main(): args.additional_pkgs, args.with_coverage, args.with_binaries, + args.fips, ) pre_build(args.clickhouse_repo_path, env_prepared) diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 1f02fdef282d..761a51e3d1ab 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -69,7 +69,7 @@ def get_packager_cmd( cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1" cmd = ( f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} " - f"--package-type={package_type} --compiler={comp}" + f"--package-type={package_type} --compiler={comp} --fips --as-root" ) if build_config["build_type"]: From 0341a10db27afbe594fd95ae8953bcb4dd583add Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 27 Jan 2023 17:47:57 +0400 Subject: [PATCH 02/11] Merge pull request #225 from Altinity/build_options Added FIPS_CLICKHOUSE to system.build_options --- src/Storages/System/StorageSystemBuildOptions.cpp.in | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/System/StorageSystemBuildOptions.cpp.in b/src/Storages/System/StorageSystemBuildOptions.cpp.in index 3465e47449b9..026eba099262 100644 --- a/src/Storages/System/StorageSystemBuildOptions.cpp.in +++ b/src/Storages/System/StorageSystemBuildOptions.cpp.in @@ -68,6 +68,7 @@ const char * auto_config_build[] "GIT_BRANCH", R"IRjaNsZIL9Yh7FQ4(@GIT_BRANCH@)IRjaNsZIL9Yh7FQ4", "GIT_DATE", "@GIT_DATE@", "GIT_COMMIT_SUBJECT", R"Gi17KJMlbGCjErEN(@GIT_COMMIT_SUBJECT@)Gi17KJMlbGCjErEN", + "FIPS_CLICKHOUSE", "@FIPS_CLICKHOUSE@", nullptr, nullptr }; From 38012ad9357d0b44dd888b9322a1a96abe5d1678 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Sun, 5 Mar 2023 14:20:06 +0100 Subject: [PATCH 03/11] Merge pull request #235 from Altinity/22.8.13-fips-go1.19-boringssl 22.8.13 BoringSSL FIPS mode --- .github/workflows/release_branches.yml | 2 +- CMakeLists.txt | 4 + cmake/autogenerated_versions.txt | 8 +- cmake/split_debug_symbols.cmake | 15 ++- contrib/boringssl-cmake/CMakeLists.txt | 120 ++++++++++++++++++ docker/packager/binary/Dockerfile | 24 ++++ docker/packager/packager | 4 +- programs/server/Server.cpp | 6 + src/Functions/FunctionShowCertificate.h | 6 +- .../System/StorageSystemCertificates.cpp | 6 +- tests/ci/build_check.py | 7 +- tests/ci/env_helper.py | 2 +- tests/ci/git_helper.py | 2 +- tests/ci/version_helper.py | 7 +- 14 files changed, 195 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index dd9693971591..3924e9a621a6 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -165,7 +165,7 @@ jobs: REPO_COPY=${{runner.temp}}/build_check/ClickHouse CACHES_PATH=${{runner.temp}}/../ccaches BUILD_NAME=package_release - CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable + CLICKHOUSE_STABLE_VERSION_SUFFIX=altinityfips EOF - name: Download changed images uses: actions/download-artifact@v3 diff --git a/CMakeLists.txt b/CMakeLists.txt index ce615c11f2b5..7ed820819982 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,10 @@ option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF) but is not possible to satisfy" ON) +option(FIPS_CLICKHOUSE + "Build ClickHouse in FIPS mode: that is both BoringSSL and Poco are build in FIPS mode" + ON) + if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION) set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR) else() diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index c9aba749240b..feab7b3e2952 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -8,9 +8,9 @@ SET(VERSION_MINOR 3) SET(VERSION_PATCH 8) SET(VERSION_GITHASH bc683c11c923c8861510410d5b8fb16933db70d9) -SET(VERSION_TWEAK 22) -SET(VERSION_FLAVOUR altinitystable) +SET(VERSION_TWEAK 23) +SET(VERSION_FLAVOUR altinityfips) -SET(VERSION_DESCRIBE v23.3.8.22.altinitystable) -SET(VERSION_STRING 23.3.8.22.altinitystable) +SET(VERSION_DESCRIBE v23.3.8.23.altinityfips) +SET(VERSION_STRING 23.3.8.23.altinityfips) # end of autochange diff --git a/cmake/split_debug_symbols.cmake b/cmake/split_debug_symbols.cmake index a9c2158359a5..e81e7f934e78 100644 --- a/cmake/split_debug_symbols.cmake +++ b/cmake/split_debug_symbols.cmake @@ -15,6 +15,19 @@ macro(clickhouse_split_debug_symbols) message(FATAL_ERROR "Destination directory for stripped binary must be provided") endif() + set(STRIP_EXTRA_ARGS "") + if (FIPS_CHLICKHOUSE) + # For FIPS tests (hash-break and to run properly, we need to keep some symbols + foreach(symbol_name IN ITEMS + BORINGSSL_bcm_rodata_start + BORINGSSL_bcm_rodata_end + BORINGSSL_bcm_text_start + BORINGSSL_bcm_text_end + ) + set(STRIP_EXTRA_ARGS "${STRIP_EXTRA_ARGS} -K ${symbol_name}") + endforeach() + endif() + add_custom_command(TARGET ${STRIP_TARGET} POST_BUILD COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/lib/debug/bin" COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/bin" @@ -23,7 +36,7 @@ macro(clickhouse_split_debug_symbols) COMMAND "${OBJCOPY_PATH}" --only-keep-debug "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}" "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug" COMMAND chmod 0644 "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug" # Strips binary, sections '.note' & '.comment' are removed in line with Debian's stripping policy: www.debian.org/doc/debian-policy/ch-files.html, section '.clickhouse.hash' is needed for integrity check: - COMMAND "${STRIP_PATH}" --remove-section=.comment --remove-section=.note --keep-section=.clickhouse.hash "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}" + COMMAND "${STRIP_PATH}" --remove-section=.comment --remove-section=.note --keep-section=.clickhouse.hash ${STRIP_EXTRA_ARGS} "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}" # Associate stripped binary with debug symbols: COMMAND "${OBJCOPY_PATH}" --add-gnu-debuglink "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug" "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}" COMMENT "Stripping clickhouse binary" VERBATIM diff --git a/contrib/boringssl-cmake/CMakeLists.txt b/contrib/boringssl-cmake/CMakeLists.txt index 828919476a77..0ccb42233e55 100644 --- a/contrib/boringssl-cmake/CMakeLists.txt +++ b/contrib/boringssl-cmake/CMakeLists.txt @@ -13,6 +13,124 @@ if(NOT ENABLE_SSL) return() endif() +if(FIPS_CLICKHOUSE) + +set(BORINGSSL_BUILD_DIR "${CMAKE_BINARY_DIR}/go1.19-boringssl-build") +set(BORINGSSL_BINARIES_DIR "${BORINGSSL_BUILD_DIR}/output") + +message("Will build BoringSSL in FIPS mode according to go1.19 recipe...") +# build BoringSSL in FIPS mode accoring to the Security Policy: +# https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp3678.pdf +# We re-use golang-1.19 routine, for that we need only 3 files from golang repo, so no need to checkout a full repository: +# - src/crypto/internal/boring/Dockerfile - build environment +# - src/crypto/internal/boring/build.sh - build and test script +# - src/crypto/internal/boring/goboringcrypto.h - required for producing golangs syso (whuch we do not need), but build will fail without it. +# hashsums were calculated on 28 Dec 2022 + +message("Downloading build files from go1.19 github...") +file(DOWNLOAD + https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/goboringcrypto.h + ${BORINGSSL_BUILD_DIR}/goboringcrypto.h + EXPECTED_HASH SHA256=2bea41082e0cc2bdfc6d5fccc64544cb52cc889e6e99330a6b423f04fef48a57 + SHOW_PROGRESS +) + +file(DOWNLOAD + https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/Dockerfile + ${BORINGSSL_BUILD_DIR}/Dockerfile + EXPECTED_HASH SHA256=c7d3d13d028f542af5dd9f173ad7b7a29bc398f61bb5dd228e6da48b05ea5487 + SHOW_PROGRESS +) + +file(DOWNLOAD + https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/build.sh + ${BORINGSSL_BUILD_DIR}/build.sh + EXPECTED_HASH SHA256=b4daa2ee2c1ce735c8720eb22d4ef21f9a7a03c737230bddf3582b0fed1b3728 + SHOW_PROGRESS +) + +# Build driver - the script that triggers the build and pulls out results from docker container +file(WRITE ${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh +"#!/bin/bash +set -ex + +OUTPUT_DIR=$1 +shift + +docker build . -t boringssl-builder +readonly id=$(docker create boringssl-builder) + +docker start -a $id #| tr -dc \\\\x0-\\\\x9 + +mkdir -p $OUTPUT_DIR + +# Copy build artifacts +docker cp $id:/boring/boringssl/build/ssl/libssl.a $OUTPUT_DIR +docker cp $id:/boring/boringssl/build/crypto/libcrypto.a $OUTPUT_DIR +docker cp $id:/boring/boringssl/build/decrepit/libdecrepit.a $OUTPUT_DIR +docker cp $id:/boring/boringssl/include $OUTPUT_DIR + +docker rm $id" +) + +# patch a single file in krb5 that relies on file missing from this version of BoringSSL +SET(krb5_filb_to_patch ${PROJECT_SOURCE_DIR}/contrib/krb5/src/lib/crypto/openssl/enc_provider/aes.c) +message("Patching ${krb5_filb_to_patch} to allow building against older version of BoringSSL") +file(READ ${krb5_filb_to_patch} FILE_CONTENTS) +string(REPLACE "#include " "//#include " FILE_CONTENTS "${FILE_CONTENTS}") +file(WRITE ${krb5_filb_to_patch} "${FILE_CONTENTS}") + +message("Creating directory for BoringSSL binaries and includes in ${BORINGSSL_BINARIES_DIR}") +execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory "${BORINGSSL_BINARIES_DIR}/include") + +add_custom_target(build-boringssl +# COMMENT "Build BoringSSL in FIPS mode with docker (using go1.19 build suite)" + DEPENDS ${BORINGSSL_BINARIES_DIR}/libssl.a ${BORINGSSL_BINARIES_DIR}/libcrypto.a +) + +add_custom_command( + OUTPUT + "${BORINGSSL_BUILD_DIR}/output/libssl.a" + "${BORINGSSL_BUILD_DIR}/output/libcrypto.a" + "${BORINGSSL_BUILD_DIR}/output/libdecrepit.a" + COMMENT "Building BoringSSL in FIPS mode using Docker" + COMMAND bash -c "chmod +x ${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh ${BORINGSSL_BUILD_DIR}/build.sh" + COMMAND bash -c "${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh ${BORINGSSL_BINARIES_DIR}" + WORKING_DIRECTORY ${BORINGSSL_BUILD_DIR} + USES_TERMINAL # To stream output + DEPENDS + ${BORINGSSL_BUILD_DIR}/build.sh + ${BORINGSSL_BUILD_DIR}/goboringcrypto.h + ${BORINGSSL_BUILD_DIR}/Dockerfile +) + +add_library(_crypto UNKNOWN IMPORTED GLOBAL) +add_dependencies(_crypto build-boringssl) +set_target_properties(_crypto PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" + IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libcrypto.a" + INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include" +) + +add_library(_decrepit UNKNOWN IMPORTED) +add_dependencies(_decrepit build-boringssl) +set_target_properties(_decrepit PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" + IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libdecrepit.a" + INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include" +) + +add_library(_ssl UNKNOWN IMPORTED GLOBAL) +add_dependencies(_ssl _crypto) +set_target_properties(_ssl PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "CXX" + IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libssl.a" + INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include" + INTERFACE_LINK_LIBRARIES _decrepit +) + +else() # FIPS_CLICKHOUSE + # Copyright (c) 2019 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -687,6 +805,8 @@ target_include_directories(_ssl SYSTEM PUBLIC "${BORINGSSL_SOURCE_DIR}/include") target_compile_options(_crypto PRIVATE -Wno-gnu-anonymous-struct) +endif() # FIPS_CLICKHOUSE + add_library(OpenSSL::Crypto ALIAS _crypto) add_library(OpenSSL::SSL ALIAS _ssl) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 9b0b9be96e90..34d6fe2751d5 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -8,6 +8,30 @@ FROM altinityinfra/test-util:$FROM_TAG as cctools RUN apt update && apt install cmake --yes +# Install Docker +ENV DOCKER_CHANNEL stable +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - +RUN add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" + +RUN apt-get update \ + && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ + docker-ce \ + && rm -rf \ + /var/lib/apt/lists/* \ + /var/cache/debconf \ + /tmp/* \ + && apt-get clean + +RUN dockerd --version; docker --version + +# Install CMake 3.20+ for Rust compilation +# Used https://askubuntu.com/a/1157132 as reference +RUN apt purge cmake --yes +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' +RUN apt update && apt install cmake --yes + + # Install Docker ENV DOCKER_CHANNEL stable RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - diff --git a/docker/packager/packager b/docker/packager/packager index b04e28fd1d41..2c32b79b64ad 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -240,7 +240,9 @@ def parse_env_variables( ) if is_release_build(build_type, package_type, sanitizer): cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON") - result.append("WITH_PERFORMANCE=1") + # NOTE(vnemkov): we don't do performace tests, and this causes some issues down the road, disabling it for now + # https://s3.amazonaws.com/altinity-build-artifacts/217/dc4053ae4a78bfff0f689f001dfb30a7fa67694d/package_release/build_log.log + # result.append("WITH_PERFORMANCE=1") if is_cross_arm: cmake_flags.append("-DBUILD_STANDALONE_KEEPER=1") else: diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 430540807c8b..a43261a489ea 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -485,6 +486,11 @@ void Server::initialize(Poco::Util::Application & self) Poco::Environment::osName(), Poco::Environment::osVersion(), Poco::Environment::osArchitecture()); + + if (FIPS_mode()) + { + LOG_INFO(&logger(), "Starting in FIPS mode, KAT test result: {}", BORINGSSL_self_test()); + } } std::string Server::getDefaultCorePath() const diff --git a/src/Functions/FunctionShowCertificate.h b/src/Functions/FunctionShowCertificate.h index 3c30d8138e5a..f96044ce1643 100644 --- a/src/Functions/FunctionShowCertificate.h +++ b/src/Functions/FunctionShowCertificate.h @@ -89,7 +89,7 @@ class FunctionShowCertificate : public IFunction { char buf[1024] = {0}; - const ASN1_INTEGER * sn = X509_get0_serialNumber(cert); + const ASN1_INTEGER * sn = X509_get_serialNumber(const_cast(cert)); BIGNUM * bnsn = ASN1_INTEGER_to_BN(sn, nullptr); SCOPE_EXIT( { @@ -116,7 +116,7 @@ class FunctionShowCertificate : public IFunction } } - char * issuer = X509_NAME_oneline(X509_get_issuer_name(cert), nullptr, 0); + char * issuer = X509_NAME_oneline(X509_get_issuer_name(const_cast(cert)), nullptr, 0); if (issuer) { SCOPE_EXIT( @@ -145,7 +145,7 @@ class FunctionShowCertificate : public IFunction } } - char * subject = X509_NAME_oneline(X509_get_subject_name(cert), nullptr, 0); + char * subject = X509_NAME_oneline(X509_get_subject_name(const_cast(cert)), nullptr, 0); if (subject) { SCOPE_EXIT( diff --git a/src/Storages/System/StorageSystemCertificates.cpp b/src/Storages/System/StorageSystemCertificates.cpp index c4d262f2f444..1492336d7e7d 100644 --- a/src/Storages/System/StorageSystemCertificates.cpp +++ b/src/Storages/System/StorageSystemCertificates.cpp @@ -55,7 +55,7 @@ static void populateTable(const X509 * cert, MutableColumns & res_columns, const { char buf[1024] = {0}; - const ASN1_INTEGER * sn = X509_get0_serialNumber(cert); + const ASN1_INTEGER * sn = X509_get_serialNumber(const_cast(cert)); BIGNUM * bnsn = ASN1_INTEGER_to_BN(sn, nullptr); SCOPE_EXIT( { @@ -83,7 +83,7 @@ static void populateTable(const X509 * cert, MutableColumns & res_columns, const } ++col; - char * issuer = X509_NAME_oneline(X509_get_issuer_name(cert), nullptr, 0); + char * issuer = X509_NAME_oneline(X509_get_issuer_name(const_cast(cert)), nullptr, 0); if (issuer) { SCOPE_EXIT( @@ -114,7 +114,7 @@ static void populateTable(const X509 * cert, MutableColumns & res_columns, const } ++col; - char * subject = X509_NAME_oneline(X509_get_subject_name(cert), nullptr, 0); + char * subject = X509_NAME_oneline(X509_get_subject_name(const_cast(cert)), nullptr, 0); if (subject) { SCOPE_EXIT( diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 761a51e3d1ab..b2c09574b576 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -314,7 +314,12 @@ def main(): official_flag = build_config["official"] official_flag = True - version._flavour = version_type = CLICKHOUSE_STABLE_VERSION_SUFFIX + # Do not override version flavour, so it must be set in one place only + if (len(CLICKHOUSE_STABLE_VERSION_SUFFIX)): + version._flavour = version_type = CLICKHOUSE_STABLE_VERSION_SUFFIX + else: + version_type = version._flavour + # TODO (vnemkov): right now we'll use simplified version management: # only update git hash and explicitly set stable version suffix. # official_flag = pr_info.number == 0 diff --git a/tests/ci/env_helper.py b/tests/ci/env_helper.py index c3ab8e797868..6d0b2233841e 100644 --- a/tests/ci/env_helper.py +++ b/tests/ci/env_helper.py @@ -25,7 +25,7 @@ S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "altinity-build-artifacts") S3_TEST_REPORTS_BUCKET = os.getenv("S3_TEST_REPORTS_BUCKET", "altinity-build-artifacts") S3_URL = os.getenv("S3_URL", "https://s3.amazonaws.com") -CLICKHOUSE_STABLE_VERSION_SUFFIX = os.getenv("CLICKHOUSE_STABLE_VERSION_SUFFIX", "stable") +CLICKHOUSE_STABLE_VERSION_SUFFIX = os.getenv("CLICKHOUSE_STABLE_VERSION_SUFFIX", "") S3_DOWNLOAD = os.getenv("S3_DOWNLOAD", S3_URL) S3_ARTIFACT_DOWNLOAD_TEMPLATE = ( diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 23914cce22c2..eefbbc01ff9b 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -14,7 +14,7 @@ # Because at early release stages there could be no Altinity tag set on commit, only upstream one. RELEASE_BRANCH_REGEXP = r"\A\d+[.]\d+\Z" TAG_REGEXP = ( - r"\Av\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*[-\.](testing|prestable|stable|lts|altinitystable)\Z" + r"\Av\d{2}[.][1-9]\d*[.][1-9]\d*[.][1-9]\d*[-\.](testing|prestable|stable|lts|altinitystable|altinityfips)\Z" ) SHA_REGEXP = r"\A([0-9]|[a-f]){40}\Z" diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 62f1e474bcc6..4c6cce4b6f39 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -31,6 +31,7 @@ SET(VERSION_GITHASH {githash}) SET(VERSION_DESCRIBE {describe}) SET(VERSION_STRING {string}) +SET(VERSION_FLAVOUR {flavour}) # end of autochange """ @@ -151,6 +152,7 @@ def as_dict(self) -> VERSIONS: "githash": self.githash, "describe": self.describe, "string": self.string, + "flavour": self._flavour } def as_tuple(self) -> Tuple[int, int, int, int]: @@ -193,11 +195,12 @@ class VersionType: PRESTABLE = "prestable" STABLE = "altinitystable" TESTING = "testing" - VALID = (TESTING, PRESTABLE, STABLE, LTS) + FIPS = "altinityfips" + VALID = (TESTING, PRESTABLE, STABLE, LTS, FIPS) def validate_version(version: str) -> None: - # NOTE(vnemkov): minor but imporant fixes, so versions with 'flavour' are threated as valid (e.g. 22.8.8.4.altinitystable) + # NOTE(vnemkov): minor but important fixes, so versions with 'flavour' are threated as valid (e.g. 22.8.8.4.altinitystable) parts = version.split(".") if len(parts) < 4: raise ValueError(f"{version} does not contain 4 parts") From 45fd28d4c126851520c4bc7610bf183c637665bc Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 18 Apr 2023 23:44:20 +0200 Subject: [PATCH 04/11] Merge pull request #255 from Altinity/keeper_in_fips_mode Keeper in fips mode --- src/Coordination/KeeperServer.cpp | 63 ++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index ca84450cf6bb..1103b31d7a4b 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -27,6 +27,15 @@ #include #include +#if USE_SSL +#include +#include + +#include +#include +#endif +#include + namespace DB { @@ -43,8 +52,50 @@ namespace { #if USE_SSL -void setSSLParams(nuraft::asio_service::options & asio_opts) + +enum class SSLContext +{ + Server, + Client +}; + +template +SSL_CTX* getSslContext() +{ + // Boring SSL states that it is Ok to use context from different threads, + // OpenSSL discourages that. +#if !defined(OPENSSL_IS_BORINGSSL) || !defined(BORINGSSL_API_VERSION) + throw Exception(ErrorCode::LOGICAL_ERROR, "Unsafe to share SSL_CTX with non-BoringSSL builds"); +#endif + + auto & ssl_manager = Poco::Net::SSLManager::instance(); + const auto & ssl_ctx = contextType == SSLContext::Server ? ssl_manager.defaultServerContext() : ssl_manager.defaultClientContext(); + + auto raw_ssl_ctx = ssl_ctx->sslContext(); + assert(raw_ssl_ctx); + + // asio will SSL_CTX_free context in desctructor, + // so we need to make sure that that wouldn't actually destroy a context. + SSL_CTX_up_ref(raw_ssl_ctx); + + return raw_ssl_ctx; +} + +void setSSLParams(nuraft::asio_service::options & asio_opts, [[maybe_unused]] Poco::Logger * log) { + // In order to maintain uniformity with CH on how SSL is used by Keeper, + // we need to use same SSL_CTX configuration as rest of ClickHouse does. + // Since copying configuration from one SSL_CTX to another is hard, we are using same + // SSL_CTX as rest of CH does (via Poco). + // + // OpenSSL explicitly prohibts sharing SSL_CTX by multiple threads, but BoringSSL allows it + // and states that SSL_CTX is thread-safe. +#if defined(OPENSSL_IS_BORINGSSL) && defined(BORINGSSL_API_VERSION) + asio_opts.ssl_context_provider_server_ = getSslContext; + asio_opts.ssl_context_provider_client_ = getSslContext; +#else + LOG_WARNING(log, "Not all openSSL cofiguration options from config are applied, some SSL-related functionalty may work incorrectly."); + const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); String certificate_file_property = "openSSL.server.certificateFile"; String private_key_file_property = "openSSL.server.privateKeyFile"; @@ -68,6 +119,7 @@ void setSSLParams(nuraft::asio_service::options & asio_opts) if (config.getString("openSSL.server.verificationMode", "none") == "none") asio_opts.skip_verification_ = true; +#endif } #endif @@ -120,6 +172,13 @@ KeeperServer::KeeperServer( if (coordination_settings->quorum_reads) LOG_WARNING(log, "Quorum reads enabled, Keeper will work slower."); +#if USE_SSL + if (FIPS_mode()) + { + LOG_INFO(log, "Starting in FIPS mode, KAT test result: {}", BORINGSSL_self_test()); + } +#endif + keeper_context->digest_enabled = config.getBool("keeper_server.digest_enabled", false); keeper_context->ignore_system_path_on_startup = config.getBool("keeper_server.ignore_system_path_on_startup", false); @@ -315,7 +374,7 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co if (state_manager->isSecure()) { #if USE_SSL - setSSLParams(asio_opts); + setSSLParams(asio_opts, log); #else throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for NuRaft is disabled because ClickHouse was built without SSL support."); #endif From 957f02603a259db574f7f4dc7fb07e9d1934948d Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 21 Jun 2023 12:34:03 +0200 Subject: [PATCH 05/11] Merge pull request #274 from Altinity/fix_keeper_coordination_ssl 22.8-FIPS Fix keeper coordination ssl --- src/Coordination/KeeperServer.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 1103b31d7a4b..32b805637283 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -65,7 +65,7 @@ SSL_CTX* getSslContext() // Boring SSL states that it is Ok to use context from different threads, // OpenSSL discourages that. #if !defined(OPENSSL_IS_BORINGSSL) || !defined(BORINGSSL_API_VERSION) - throw Exception(ErrorCode::LOGICAL_ERROR, "Unsafe to share SSL_CTX with non-BoringSSL builds"); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unsafe to share SSL_CTX with non-BoringSSL builds"); #endif auto & ssl_manager = Poco::Net::SSLManager::instance(); @@ -91,6 +91,7 @@ void setSSLParams(nuraft::asio_service::options & asio_opts, [[maybe_unused]] Po // OpenSSL explicitly prohibts sharing SSL_CTX by multiple threads, but BoringSSL allows it // and states that SSL_CTX is thread-safe. #if defined(OPENSSL_IS_BORINGSSL) && defined(BORINGSSL_API_VERSION) + asio_opts.enable_ssl_ = true; asio_opts.ssl_context_provider_server_ = getSslContext; asio_opts.ssl_context_provider_client_ = getSslContext; #else @@ -379,6 +380,10 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSL support for NuRaft is disabled because ClickHouse was built without SSL support."); #endif } + else if (FIPS_mode() && !asio_opts.enable_ssl_) + { + LOG_FATAL(log, "Can't start NON-SECURE keeper server in FIPS mode, please check the config."); + } if (is_recovering) enterRecoveryMode(params); From eb028a44747c2fbb110a1b556a531964bc35d782 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 3 Jul 2023 12:40:19 +0200 Subject: [PATCH 06/11] Attempt to fix docker-in-docker setup for packager --- docker/packager/binary/Dockerfile | 64 ++++++++----------------------- 1 file changed, 16 insertions(+), 48 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 34d6fe2751d5..05bce35ff824 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -7,47 +7,6 @@ FROM altinityinfra/test-util:$FROM_TAG as cctools # Theoretically, it should improve rebuild speed significantly RUN apt update && apt install cmake --yes - -# Install Docker -ENV DOCKER_CHANNEL stable -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -RUN add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" - -RUN apt-get update \ - && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ - docker-ce \ - && rm -rf \ - /var/lib/apt/lists/* \ - /var/cache/debconf \ - /tmp/* \ - && apt-get clean - -RUN dockerd --version; docker --version - -# Install CMake 3.20+ for Rust compilation -# Used https://askubuntu.com/a/1157132 as reference -RUN apt purge cmake --yes -RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null -RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' -RUN apt update && apt install cmake --yes - - -# Install Docker -ENV DOCKER_CHANNEL stable -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -RUN add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" - -RUN apt-get update \ - && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ - docker-ce \ - && rm -rf \ - /var/lib/apt/lists/* \ - /var/cache/debconf \ - /tmp/* \ - && apt-get clean - -RUN dockerd --version; docker --version - ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} # !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -80,6 +39,22 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ && cd ../.. \ && rm -rf cctools-port +# !!!!!!!!!!! +# END COMPILE +# !!!!!!!!!!! + +FROM altinityinfra/test-util:$FROM_TAG +ENV CC=clang-${LLVM_VERSION} +ENV CXX=clang++-${LLVM_VERSION} + +# Install CMake 3.20+ for Rust compilation +# Used https://askubuntu.com/a/1157132 as reference +RUN apt purge cmake --yes +RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null +RUN apt-add-repository 'deb https://apt.kitware.com/ubuntu/ focal main' +RUN apt update && apt install cmake --yes + + # Install Docker ENV DOCKER_CHANNEL stable RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - @@ -96,13 +71,6 @@ RUN apt-get update \ RUN dockerd --version; docker --version -# !!!!!!!!!!! -# END COMPILE -# !!!!!!!!!!! - -FROM altinityinfra/test-util:$FROM_TAG -ENV CC=clang-${LLVM_VERSION} -ENV CXX=clang++-${LLVM_VERSION} # Rust toolchain and libraries ENV RUSTUP_HOME=/rust/rustup From 6f0157a7745c96b4d3813f0b0b1a1856b903652d Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 3 Jul 2023 14:05:43 +0200 Subject: [PATCH 07/11] Attempt to fix dockerd startup in packager --- docker/packager/binary/Dockerfile | 7 +++---- docker/packager/binary/build.sh | 6 +++++- docker/test/integration/runner/dockerd-entrypoint.sh | 3 ++- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 05bce35ff824..f0d0bd91a292 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -57,10 +57,9 @@ RUN apt update && apt install cmake --yes # Install Docker ENV DOCKER_CHANNEL stable -RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - -RUN add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" - -RUN apt-get update \ +RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add - \ + && add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -c -s) ${DOCKER_CHANNEL}" \ + && apt-get update \ && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ docker-ce \ && rm -rf \ diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 242fb2267dbb..3e87524af13b 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -6,7 +6,10 @@ set -e # and on hung you can simply press Ctrl-C and it will spawn a python pdb, # but on SIGINT dockerd will exit, so ignore it to preserve the daemon. trap '' INT -dockerd --host=unix:///var/run/docker.sock --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 & +# Binding to an IP address without --tlsverify is deprecated. Startup is intentionally being slowed +# unless --tls=false or --tlsverify=false is set +dockerd --host=unix:///var/run/docker.sock --tls=false --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 \ + &> /ClickHouse/packager/dockerd.log & set +e reties=0 @@ -15,6 +18,7 @@ while true; do reties=$((reties+1)) if [[ $reties -ge 100 ]]; then # 10 sec max echo "Can't start docker daemon, timeout exceeded." >&2 + cat /ClickHouse/packager/dockerd.log >&2 exit 1; fi sleep 0.1 diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index 4394b42a9e98..8da47dc87bb7 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -18,7 +18,8 @@ echo '{ trap '' INT # Binding to an IP address without --tlsverify is deprecated. Startup is intentionally being slowed # unless --tls=false or --tlsverify=false is set -dockerd --host=unix:///var/run/docker.sock --tls=false --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 &>/ClickHouse/tests/integration/dockerd.log & +dockerd --host=unix:///var/run/docker.sock --tls=false --host=tcp://0.0.0.0:2375 --default-address-pool base=172.17.0.0/12,size=24 \ + &>/ClickHouse/tests/integration/dockerd.log & set +e reties=0 From 1e89979d62d2b4990f1672c1ab9e96d0547c7b50 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Mon, 3 Jul 2023 18:45:20 +0200 Subject: [PATCH 08/11] Fixed building --- docker/packager/binary/Dockerfile | 36 +++++++++++++++---------------- docker/packager/binary/build.sh | 13 +++++------ docker/packager/packager | 1 - 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index f0d0bd91a292..cc5e6e944561 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -5,7 +5,6 @@ ARG FROM_TAG=latest FROM altinityinfra/test-util:$FROM_TAG as cctools # The cctools are built always from the altinityinfra/test-util:latest and cached inline # Theoretically, it should improve rebuild speed significantly -RUN apt update && apt install cmake --yes ENV CC=clang-${LLVM_VERSION} ENV CXX=clang++-${LLVM_VERSION} @@ -21,23 +20,24 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/apple-libtapi.git \ && cd .. \ && rm -rf apple-libtapi -# Build and install tools for cross-linking to Darwin (x86-64) -RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ - && cd cctools-port/cctools \ - && ./configure --prefix=/cctools --with-libtapi=/cctools \ - --target=x86_64-apple-darwin \ - && make install -j$(nproc) \ - && cd ../.. \ - && rm -rf cctools-port - -# Build and install tools for cross-linking to Darwin (aarch64) -RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ - && cd cctools-port/cctools \ - && ./configure --prefix=/cctools --with-libtapi=/cctools \ - --target=aarch64-apple-darwin \ - && make install -j$(nproc) \ - && cd ../.. \ - && rm -rf cctools-port +# NOTE(vnemkov): Not required for arm64 only build, reenable for arm64 builds. +# # Build and install tools for cross-linking to Darwin (x86-64) +# RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ +# && cd cctools-port/cctools \ +# && ./configure --prefix=/cctools --with-libtapi=/cctools \ +# --target=x86_64-apple-darwin \ +# && make install -j$(nproc) \ +# && cd ../.. \ +# && rm -rf cctools-port + +# # Build and install tools for cross-linking to Darwin (aarch64) +# RUN git clone --depth 1 https://github.com/tpoechtrager/cctools-port.git \ +# && cd cctools-port/cctools \ +# && ./configure --prefix=/cctools --with-libtapi=/cctools \ +# --target=aarch64-apple-darwin \ +# && make install -j$(nproc) \ +# && cd ../.. \ +# && rm -rf cctools-port # !!!!!!!!!!! # END COMPILE diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 3e87524af13b..65b85ebc4944 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -65,12 +65,13 @@ if [ -n "$MAKE_DEB" ]; then if [ -d /build/programs/diagnostics ]; then if [ -z "$SANITIZER" ]; then # We need to check if clickhouse-diagnostics is fine and build it - ( - cd /build/programs/diagnostics - make test-no-docker - GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build - mv clickhouse-diagnostics .. - ) + # ( + # cd /build/programs/diagnostics + # make test-no-docker + # GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build + # mv clickhouse-diagnostics .. + # ) + echo "NOT building clickhouse-diagnostics else echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics chmod +x /build/programs/clickhouse-diagnostics diff --git a/docker/packager/packager b/docker/packager/packager index 2c32b79b64ad..1ea80a352557 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -216,7 +216,6 @@ def parse_env_variables( result.append("DEB_ARCH=amd64") if fips: - cmake_flags.append("-DLINKER_NAME=/usr/lib/llvm-14/bin/ld.lld") cmake_flags.append("-DFIPS_CLICKHOUSE=ON") cxx = cc.replace("gcc", "g++").replace("clang", "clang++") From de6aed26aeb45b26b6c03fd633ccd6bc815cbf20 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 4 Jul 2023 13:40:24 +0200 Subject: [PATCH 09/11] Fixed missing closing quoite --- docker/packager/binary/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 65b85ebc4944..3e1fa7dc5e9f 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -71,7 +71,7 @@ if [ -n "$MAKE_DEB" ]; then # GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build # mv clickhouse-diagnostics .. # ) - echo "NOT building clickhouse-diagnostics + echo "NOT building clickhouse-diagnostics" else echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics chmod +x /build/programs/clickhouse-diagnostics From ae95c8ccd89a176c647351f25f86b40221c74f2b Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Tue, 4 Jul 2023 15:57:56 +0200 Subject: [PATCH 10/11] A dummy script instead of clickhouse-diagnostics --- docker/packager/binary/build.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/packager/binary/build.sh b/docker/packager/binary/build.sh index 3e1fa7dc5e9f..669ba847a3aa 100755 --- a/docker/packager/binary/build.sh +++ b/docker/packager/binary/build.sh @@ -63,7 +63,7 @@ if [ -n "$MAKE_DEB" ]; then # NOTE: this is for backward compatibility with previous releases, # that does not diagnostics tool (only script). if [ -d /build/programs/diagnostics ]; then - if [ -z "$SANITIZER" ]; then + # if [ -z "$SANITIZER" ]; then # We need to check if clickhouse-diagnostics is fine and build it # ( # cd /build/programs/diagnostics @@ -71,11 +71,11 @@ if [ -n "$MAKE_DEB" ]; then # GOARCH="${DEB_ARCH}" CGO_ENABLED=0 make VERSION="$VERSION_STRING" build # mv clickhouse-diagnostics .. # ) - echo "NOT building clickhouse-diagnostics" - else + # echo "NOT building clickhouse-diagnostics" + # else echo -e "#!/bin/sh\necho 'Not implemented for this type of package'" > /build/programs/clickhouse-diagnostics chmod +x /build/programs/clickhouse-diagnostics - fi + # fi fi fi From d84894637f5f547da467bbbc15faab2ae8b44ffb Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Wed, 6 Mar 2024 14:49:07 +0000 Subject: [PATCH 11/11] Fixed build error related to logger, also fixed some typos --- src/Coordination/KeeperServer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 696b57abe96b..5c4c47de9965 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -81,7 +81,7 @@ SSL_CTX* getSslContext() return raw_ssl_ctx; } -void setSSLParams(nuraft::asio_service::options & asio_opts, [[maybe_unused]] Poco::Logger * log) +void setSSLParams(nuraft::asio_service::options & asio_opts, [[maybe_unused]] LoggerPtr log) { // In order to maintain uniformity with CH on how SSL is used by Keeper, // we need to use same SSL_CTX configuration as rest of ClickHouse does. @@ -95,7 +95,7 @@ void setSSLParams(nuraft::asio_service::options & asio_opts, [[maybe_unused]] Po asio_opts.ssl_context_provider_server_ = getSslContext; asio_opts.ssl_context_provider_client_ = getSslContext; #else - LOG_WARNING(log, "Not all openSSL cofiguration options from config are applied, some SSL-related functionalty may work incorrectly."); + LOG_WARNING(log, "Not all openSSL configuration options from config are applied, some SSL-related functionality may work incorrectly."); const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config(); String certificate_file_property = "openSSL.server.certificateFile";