23.3.19 FIPS Pre-release

.github/workflows/release_branches.yml

@@ -140,7 +140,7 @@ jobs:
       timeout_minutes: 180
       runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce
       additional_envs: |
-        CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
+        CLICKHOUSE_STABLE_VERSION_SUFFIX=altinityfips
 
   BuilderDebAarch64:
     needs: [DockerHubPush]
@@ -152,7 +152,7 @@ jobs:
       timeout_minutes: 180
       runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce
       additional_envs: |
-        CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
+        CLICKHOUSE_STABLE_VERSION_SUFFIX=altinityfips
 
   BuilderDebAsan:
     needs: [DockerHubPush]
@@ -177,7 +177,7 @@ jobs:
       runner_type: altinity-on-demand, altinity-type-ccx53, altinity-in-ash, altinity-image-x86-app-docker-ce
       additional_envs: |
         CLICKHOUSE_STABLE_VERSION_SUFFIX=altinitystable
-        
+
   BuilderDebTsan:
     needs: [DockerHubPush]
     uses: ./.github/workflows/reusable_build.yml
@@ -353,7 +353,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatelessTestTsan:
     needs: [BuilderDebTsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -367,7 +367,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatelessTestUBsan:
     needs: [BuilderDebUBsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -381,7 +381,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatelessTestMsan:
     needs: [BuilderDebMsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -395,7 +395,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatelessTestDebug:
     needs: [BuilderDebDebug]
     uses: ./.github/workflows/reusable_test.yml
@@ -451,7 +451,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatefulTestTsan:
     needs: [BuilderDebTsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -464,7 +464,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatefulTestMsan:
     needs: [BuilderDebMsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -477,7 +477,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatefulTestUBsan:
     needs: [BuilderDebUBsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -490,7 +490,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
   FunctionalStatefulTestDebug:
     needs: [BuilderDebDebug]
     uses: ./.github/workflows/reusable_test.yml
@@ -503,7 +503,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT"
-        
+
 ##############################################################################################
 ######################################### STRESS TESTS #######################################
 ##############################################################################################
@@ -517,7 +517,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 stress_check.py "$CHECK_NAME"
-        
+
   StressTestTsan:
     needs: [BuilderDebTsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -528,7 +528,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 stress_check.py "$CHECK_NAME"
-        
+
   StressTestMsan:
     needs: [BuilderDebMsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -539,7 +539,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 stress_check.py "$CHECK_NAME"
-        
+
   StressTestUBsan:
     needs: [BuilderDebUBsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -550,7 +550,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 stress_check.py "$CHECK_NAME"
-        
+
   StressTestDebug:
     needs: [BuilderDebDebug]
     uses: ./.github/workflows/reusable_test.yml
@@ -577,7 +577,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 integration_test_check.py "$CHECK_NAME"
-        
+
   IntegrationTestsAnalyzerAsan:
     needs: [BuilderDebAsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -590,7 +590,7 @@ jobs:
       run_command: |
         cd "$REPO_COPY/tests/ci"
         python3 integration_test_check.py "$CHECK_NAME"
-        
+
   IntegrationTestsTsan:
     needs: [BuilderDebTsan]
     uses: ./.github/workflows/reusable_test.yml
@@ -627,7 +627,7 @@ jobs:
     with:
       runner_type: altinity-on-demand, altinity-type-cpx51, altinity-image-x86-app-docker-ce, altinity-setup-regression
       commit: 6da94b78dc53cb8965ab56c04a89ebf54ed04cbc
-      arch: release 
+      arch: release
       build_sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.event_name == 'release' && github.sha }}
 
   RegressionTestsAarch64:

CMakeLists.txt

@@ -7,6 +7,10 @@ option(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION
    "Stop/Fail CMake configuration if some ENABLE_XXX option is defined (either ON or OFF)
    but is not possible to satisfy" ON)
 
+option(FIPS_CLICKHOUSE
+    "Build ClickHouse in FIPS mode: that is both BoringSSL and Poco are build in FIPS mode"
+    ON)
+
 if(FAIL_ON_UNSUPPORTED_OPTIONS_COMBINATION)
     set(RECONFIGURE_MESSAGE_LEVEL FATAL_ERROR)
 else()

cmake/autogenerated_versions.txt

@@ -8,9 +8,9 @@ SET(VERSION_MINOR 3)
 SET(VERSION_PATCH 19)
 SET(VERSION_GITHASH 7228475d77afaf8a59d489694343593d3b650170)
 
-SET(VERSION_TWEAK 33)
-SET(VERSION_FLAVOUR altinitystable)
+SET(VERSION_TWEAK 34)
+SET(VERSION_FLAVOUR altinityfips)
 
-SET(VERSION_DESCRIBE v23.3.19.33.altinitystable)
-SET(VERSION_STRING 23.3.19.33.altinitystable)
+SET(VERSION_DESCRIBE v23.3.19.34.altinityfips)
+SET(VERSION_STRING 23.3.19.34.altinityfips)
 # end of autochange

cmake/split_debug_symbols.cmake

@@ -15,6 +15,19 @@ macro(clickhouse_split_debug_symbols)
        message(FATAL_ERROR "Destination directory for stripped binary must be provided")
    endif()
 
+   set(STRIP_EXTRA_ARGS "")
+   if (FIPS_CHLICKHOUSE)
+       # For FIPS tests (hash-break and to run properly, we need to keep some symbols
+       foreach(symbol_name IN ITEMS
+               BORINGSSL_bcm_rodata_start
+               BORINGSSL_bcm_rodata_end
+               BORINGSSL_bcm_text_start
+               BORINGSSL_bcm_text_end
+        )
+           set(STRIP_EXTRA_ARGS "${STRIP_EXTRA_ARGS} -K ${symbol_name}")
+       endforeach()
+   endif()
+
    add_custom_command(TARGET ${STRIP_TARGET} POST_BUILD
        COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/lib/debug/bin"
        COMMAND mkdir -p "${STRIP_DESTINATION_DIR}/bin"
@@ -23,7 +36,7 @@ macro(clickhouse_split_debug_symbols)
        COMMAND "${OBJCOPY_PATH}" --only-keep-debug "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}" "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug"
        COMMAND chmod 0644 "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug"
        # Strips binary, sections '.note' & '.comment' are removed in line with Debian's stripping policy: www.debian.org/doc/debian-policy/ch-files.html, section '.clickhouse.hash' is needed for integrity check:
-       COMMAND "${STRIP_PATH}" --remove-section=.comment --remove-section=.note --keep-section=.clickhouse.hash "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
+       COMMAND "${STRIP_PATH}" --remove-section=.comment --remove-section=.note --keep-section=.clickhouse.hash ${STRIP_EXTRA_ARGS} "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
        # Associate stripped binary with debug symbols:
        COMMAND "${OBJCOPY_PATH}" --add-gnu-debuglink "${STRIP_DESTINATION_DIR}/lib/debug/bin/${STRIP_TARGET}.debug" "${STRIP_DESTINATION_DIR}/bin/${STRIP_TARGET}"
        COMMENT "Stripping clickhouse binary" VERBATIM

contrib/boringssl-cmake/CMakeLists.txt

@@ -13,6 +13,124 @@ if(NOT ENABLE_SSL)
   return()
 endif()
 
+if(FIPS_CLICKHOUSE)
+
+set(BORINGSSL_BUILD_DIR "${CMAKE_BINARY_DIR}/go1.19-boringssl-build")
+set(BORINGSSL_BINARIES_DIR "${BORINGSSL_BUILD_DIR}/output")
+
+message("Will build BoringSSL in FIPS mode according to go1.19 recipe...")
+# build BoringSSL in FIPS mode accoring to the Security Policy:
+# https://csrc.nist.gov/CSRC/media/projects/cryptographic-module-validation-program/documents/security-policies/140sp3678.pdf
+# We re-use golang-1.19 routine, for that we need only 3 files from golang repo, so no need to checkout a full repository:
+# - src/crypto/internal/boring/Dockerfile - build environment
+# - src/crypto/internal/boring/build.sh - build and test script
+# - src/crypto/internal/boring/goboringcrypto.h - required for producing golangs syso (whuch we do not need), but build will fail without it.
+# hashsums were calculated on 28 Dec 2022
+
+message("Downloading build files from go1.19 github...")
+file(DOWNLOAD
+    https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/goboringcrypto.h
+    ${BORINGSSL_BUILD_DIR}/goboringcrypto.h
+    EXPECTED_HASH SHA256=2bea41082e0cc2bdfc6d5fccc64544cb52cc889e6e99330a6b423f04fef48a57
+    SHOW_PROGRESS
+)
+
+file(DOWNLOAD
+    https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/Dockerfile
+    ${BORINGSSL_BUILD_DIR}/Dockerfile
+    EXPECTED_HASH SHA256=c7d3d13d028f542af5dd9f173ad7b7a29bc398f61bb5dd228e6da48b05ea5487
+    SHOW_PROGRESS
+)
+
+file(DOWNLOAD
+    https://raw.githubusercontent.com/golang/go/go1.19/src/crypto/internal/boring/build.sh
+    ${BORINGSSL_BUILD_DIR}/build.sh
+    EXPECTED_HASH SHA256=b4daa2ee2c1ce735c8720eb22d4ef21f9a7a03c737230bddf3582b0fed1b3728
+    SHOW_PROGRESS
+)
+
+# Build driver - the script that triggers the build and pulls out results from docker container
+file(WRITE ${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh
+"#!/bin/bash
+set -ex
+
+OUTPUT_DIR=$1
+shift
+
+docker build . -t boringssl-builder
+readonly id=$(docker create boringssl-builder)
+
+docker start -a $id #| tr -dc \\\\x0-\\\\x9
+
+mkdir -p $OUTPUT_DIR
+
+# Copy build artifacts
+docker cp $id:/boring/boringssl/build/ssl/libssl.a $OUTPUT_DIR
+docker cp $id:/boring/boringssl/build/crypto/libcrypto.a $OUTPUT_DIR
+docker cp $id:/boring/boringssl/build/decrepit/libdecrepit.a $OUTPUT_DIR
+docker cp $id:/boring/boringssl/include $OUTPUT_DIR
+
+docker rm $id"
+)
+
+# patch a single file in krb5 that relies on file missing from this version of BoringSSL
+SET(krb5_filb_to_patch ${PROJECT_SOURCE_DIR}/contrib/krb5/src/lib/crypto/openssl/enc_provider/aes.c)
+message("Patching ${krb5_filb_to_patch} to allow building against older version of BoringSSL")
+file(READ ${krb5_filb_to_patch} FILE_CONTENTS)
+string(REPLACE "#include <openssl/modes.h>" "//#include <openssl/modes.h>" FILE_CONTENTS "${FILE_CONTENTS}")
+file(WRITE ${krb5_filb_to_patch} "${FILE_CONTENTS}")
+
+message("Creating directory for BoringSSL binaries and includes in ${BORINGSSL_BINARIES_DIR}")
+execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory "${BORINGSSL_BINARIES_DIR}/include")
+
+add_custom_target(build-boringssl
+#    COMMENT "Build BoringSSL in FIPS mode with docker (using go1.19 build suite)"
+    DEPENDS ${BORINGSSL_BINARIES_DIR}/libssl.a ${BORINGSSL_BINARIES_DIR}/libcrypto.a
+)
+
+add_custom_command(
+    OUTPUT
+        "${BORINGSSL_BUILD_DIR}/output/libssl.a"
+        "${BORINGSSL_BUILD_DIR}/output/libcrypto.a"
+        "${BORINGSSL_BUILD_DIR}/output/libdecrepit.a"
+    COMMENT "Building BoringSSL in FIPS mode using Docker"
+    COMMAND bash -c "chmod +x ${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh ${BORINGSSL_BUILD_DIR}/build.sh"
+    COMMAND bash -c "${BORINGSSL_BUILD_DIR}/build_boringssl_fips.sh ${BORINGSSL_BINARIES_DIR}"
+    WORKING_DIRECTORY ${BORINGSSL_BUILD_DIR}
+    USES_TERMINAL # To stream output
+    DEPENDS
+        ${BORINGSSL_BUILD_DIR}/build.sh
+        ${BORINGSSL_BUILD_DIR}/goboringcrypto.h
+        ${BORINGSSL_BUILD_DIR}/Dockerfile
+)
+
+add_library(_crypto UNKNOWN IMPORTED GLOBAL)
+add_dependencies(_crypto build-boringssl)
+set_target_properties(_crypto PROPERTIES
+    IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+    IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libcrypto.a"
+    INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include"
+)
+
+add_library(_decrepit UNKNOWN IMPORTED)
+add_dependencies(_decrepit build-boringssl)
+set_target_properties(_decrepit PROPERTIES
+    IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+    IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libdecrepit.a"
+    INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include"
+)
+
+add_library(_ssl UNKNOWN IMPORTED GLOBAL)
+add_dependencies(_ssl _crypto)
+set_target_properties(_ssl PROPERTIES
+    IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
+    IMPORTED_LOCATION "${BORINGSSL_BINARIES_DIR}/libssl.a"
+    INTERFACE_INCLUDE_DIRECTORIES "${BORINGSSL_BINARIES_DIR}/include"
+    INTERFACE_LINK_LIBRARIES _decrepit
+)
+
+else() # FIPS_CLICKHOUSE
+
 # Copyright (c) 2019 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
@@ -687,6 +805,8 @@ target_include_directories(_ssl SYSTEM PUBLIC "${BORINGSSL_SOURCE_DIR}/include")
 
 target_compile_options(_crypto PRIVATE -Wno-gnu-anonymous-struct)
 
+endif() # FIPS_CLICKHOUSE
+
 add_library(OpenSSL::Crypto ALIAS _crypto)
 add_library(OpenSSL::SSL ALIAS _ssl)