support sm4 by gmssl

.gitmodules

@@ -89,3 +89,6 @@
 [submodule "contrib/magic_enum"]
 	path = contrib/magic_enum
 	url = https://github.com/Neargye/magic_enum.git
+[submodule "contrib/GmSSL"]
+	path = contrib/GmSSL
+	url = https://github.com/guanzhi/GmSSL.git

cmake/find_ssl.cmake

@@ -14,11 +14,7 @@
 
 # Default value on mac is ON to make development easy.
 # On other platform it is OFF because we should release tiflash binary built with openssl.
-if (APPLE)
-    option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
-else()
-    option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" OFF)
-endif()
+option(USE_INTERNAL_SSL_LIBRARY "Set to FALSE to use system *ssl library instead of bundled" ${NOT_UNBUNDLED})
 
 if(NOT EXISTS "${TiFlash_SOURCE_DIR}/contrib/boringssl/README.md")
     if(USE_INTERNAL_SSL_LIBRARY)
@@ -29,6 +25,13 @@ if(NOT EXISTS "${TiFlash_SOURCE_DIR}/contrib/boringssl/README.md")
     set(MISSING_INTERNAL_SSL_LIBRARY 1)
 endif()
 
+if (NOT APPLE)
+    option(USE_GM_SSL "Set to FALSE to disable GmSSL" ${USE_INTERNAL_SSL_LIBRARY})
+else()
+    # Avoid link to GmSSL when compile on macos because GmSSL only supports dynamic link which complicate the binary package
+    option(USE_GM_SSL "Set to FALSE to disable GmSSL" 0)
+endif()
+
 set (OPENSSL_USE_STATIC_LIBS ${USE_STATIC_LIBRARIES})
 
 if (NOT USE_INTERNAL_SSL_LIBRARY)
@@ -139,4 +142,4 @@ if(OPENSSL_FOUND AND NOT USE_INTERNAL_SSL_LIBRARY)
     endif()
 endif()
 
-message (STATUS "Using ssl=${USE_SSL}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}")
+message (STATUS "Using ssl=${USE_SSL} gmssl=${USE_GM_SSL}: ${OPENSSL_INCLUDE_DIR} : ${OPENSSL_LIBRARIES}")

contrib/CMakeLists.txt

@@ -60,6 +60,19 @@ if (USE_INTERNAL_SSL_LIBRARY)
     add_library (OpenSSL::SSL ALIAS ssl)
 endif ()
 
+if (USE_GM_SSL)
+    set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
+    set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})
+    set (_save ${ENABLE_TESTS})
+    set (CMAKE_CXX_FLAGS "")
+    set (CMAKE_C_FLAGS "")
+    set (ENABLE_TESTS 0)
+    add_subdirectory (GmSSL)
+    set (ENABLE_TESTS ${_save})
+    set (CMAKE_CXX_FLAGS ${save_CMAKE_CXX_FLAGS})
+    set (CMAKE_C_FLAGS ${save_CMAKE_C_FLAGS})
+endif()
+
 if (USE_INTERNAL_POCO_LIBRARY)
     set (save_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
     set (save_CMAKE_C_FLAGS ${CMAKE_C_FLAGS})

dbms/CMakeLists.txt

@@ -215,6 +215,12 @@ target_link_libraries (dbms
     absl::synchronization
 )
 
+# always add GmSSL include dir to the include path for static analysis
+target_include_directories(dbms PRIVATE ${TiFlash_SOURCE_DIR}/contrib/GmSSL/include)
+if (USE_GM_SSL)
+    target_link_libraries(dbms gmssl)
+endif ()
+
 if (NOT USE_INTERNAL_RE2_LIBRARY)
     target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
 endif ()
@@ -310,6 +316,33 @@ if (ENABLE_TESTS)
 
     target_link_libraries(gtests_dbms test_util_gtest_main clickhouse_functions tiflash-dttool-lib)
 
+    install (TARGETS gtests_dbms
+            COMPONENT tiflash-gtest
+            DESTINATION "."
+            RUNTIME_DEPENDENCY_SET tiflash-gtest-dependency)
+    install (RUNTIME_DEPENDENCY_SET tiflash-gtest-dependency
+            COMPONENT tiflash-gtest
+            DESTINATION "."
+            PRE_EXCLUDE_REGEXES
+                "libdl.*"
+                "libc-.*"
+                "libc\\..*"
+                "libgcc_s.*"
+                "librt.*"
+                "libm.*"
+                "ld-linux-x86-64.*"
+                "ld-linux-aarch64.*"
+                "libpthread.*"# exclude libc dependencies
+            DIRECTORIES ${TIFLASH_GTEST_DEPENDENCY_DIRECTORIES})
+
+    if (USE_GM_SSL)
+        target_link_libraries(gtests_dbms gmssl)
+        target_include_directories(gtests_dbms PRIVATE ${TiFlash_SOURCE_DIR}/contrib/GmSSL/include)
+        install (TARGETS gmssl
+                COMPONENT tiflash-gtest
+                DESTINATION ".")
+    endif ()
+
     target_compile_options(gtests_dbms PRIVATE -Wno-unknown-pragmas -Wno-deprecated-copy)
     add_check(gtests_dbms)
 

dbms/src/Common/TiFlashBuildInfo.cpp

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#include <Common/config.h>
 #include <Common/config_version.h>
 #include <common/config_common.h>
 #include <fmt/core.h>
@@ -66,8 +67,10 @@ std::string getEnabledFeatures()
 #endif
 
 // sm4
-#if OPENSSL_VERSION_NUMBER >= 0x1010100fL && !defined(OPENSSL_NO_SM4)
-            "sm4",
+#if USE_GM_SSL
+            "sm4(GmSSL)",
+#elif OPENSSL_VERSION_NUMBER >= 0x1010100fL && !defined(OPENSSL_NO_SM4)
+            "sm4(OpenSSL)",
 #endif
 
 // mem-profiling

dbms/src/Common/config.h.in

@@ -6,3 +6,4 @@
 #cmakedefine01 USE_RE2_ST
 #cmakedefine01 USE_VECTORCLASS
 #cmakedefine01 Poco_NetSSL_FOUND
+#cmakedefine01 USE_GM_SSL

dbms/src/Encryption/AESCTRCipherStream.cpp

@@ -19,7 +19,6 @@
 #include <Storages/Transaction/FileEncryption.h>
 
 #include <cassert>
-#include <cstddef>
 #include <ext/scope_guard.h>
 #include <limits>
 
@@ -41,13 +40,7 @@ size_t keySize(EncryptionMethod method)
     case EncryptionMethod::Aes256Ctr:
         return 32;
     case EncryptionMethod::SM4Ctr:
-#if OPENSSL_VERSION_NUMBER < 0x1010100fL || defined(OPENSSL_NO_SM4)
-        throw DB::TiFlashException("Unsupported encryption method: " + std::to_string(static_cast<int>(method)),
-                                   Errors::Encryption::Internal);
-#else
-        // OpenSSL support SM4 after 1.1.1 release version.
         return 16;
-#endif
     default:
         return 0;
     }
@@ -62,12 +55,11 @@ size_t blockSize(EncryptionMethod method)
     case EncryptionMethod::Aes256Ctr:
         return AES_BLOCK_SIZE;
     case EncryptionMethod::SM4Ctr:
-#if OPENSSL_VERSION_NUMBER < 0x1010100fL || defined(OPENSSL_NO_SM4)
+#if defined(SM4_BLOCK_SIZE)
+        return SM4_BLOCK_SIZE;
+#else
         throw DB::TiFlashException("Unsupported encryption method: " + std::to_string(static_cast<int>(method)),
                                    Errors::Encryption::Internal);
-#else
-        // OpenSSL support SM4 after 1.1.1 release version.
-        return SM4_BLOCK_SIZE;
 #endif
     default:
         return 0;
@@ -83,45 +75,40 @@ void AESCTRCipherStream::cipher(uint64_t file_offset, char * data, size_t data_s
     (void)is_encrypt;
     throw Exception("OpenSSL version < 1.0.2", ErrorCodes::NOT_IMPLEMENTED);
 #else
-    int ret = 1;
-    EVP_CIPHER_CTX * ctx = nullptr;
-    InitCipherContext(ctx);
-    RUNTIME_CHECK_MSG(ctx != nullptr, "Failed to create cipher context.");
-
-    SCOPE_EXIT({ FreeCipherContext(ctx); });
-
     const size_t block_size = blockSize();
     uint64_t block_index = file_offset / block_size;
     uint64_t block_offset = file_offset % block_size;
-
-    // In CTR mode, OpenSSL EVP API treat the IV as a 128-bit big-endian, and
-    // increase it by 1 for each block.
-    uint64_t iv_high = initial_iv_high_;
-    uint64_t iv_low = initial_iv_low_ + block_index;
-    if (std::numeric_limits<uint64_t>::max() - block_index < initial_iv_low_)
-    {
-        iv_high++;
-    }
-    iv_high = toBigEndian(iv_high);
-    iv_low = toBigEndian(iv_low);
     unsigned char iv[block_size];
-    memcpy(iv, &iv_high, sizeof(uint64_t));
-    memcpy(iv + sizeof(uint64_t), &iv_low, sizeof(uint64_t));
-
-    ret = EVP_CipherInit(ctx, cipher_, reinterpret_cast<const unsigned char *>(key_.data()), iv, (is_encrypt ? 1 : 0));
-    RUNTIME_CHECK_MSG(ret == 1, "Failed to create cipher context.");
-
-    // Disable padding. After disabling padding, data size should always be
-    // multiply of block size.
-    ret = EVP_CIPHER_CTX_set_padding(ctx, 0);
-    RUNTIME_CHECK_MSG(ret == 1, "Failed to disable padding for cipher context.");
+    initIV(block_index, iv);
 
     uint64_t data_offset = 0;
     size_t remaining_data_size = data_size;
     int output_size = 0;
     unsigned char partial_block[block_size];
 
-    // In the following we assume EVP_CipherUpdate allow in and out buffer are
+    int ret = 1;
+    EVP_CIPHER_CTX * ctx = nullptr;
+    InitCipherContext(ctx);
+    RUNTIME_CHECK_MSG(ctx != nullptr, "Failed to create cipher context.");
+    SCOPE_EXIT({ FreeCipherContext(ctx); });
+
+
+#if !USE_GM_SSL
+    RUNTIME_CHECK_MSG(cipher_ != nullptr, "Cipher is not valid.");
+#endif
+
+    if (cipher_ != nullptr)
+    {
+        ret = EVP_CipherInit(ctx, cipher_, reinterpret_cast<const unsigned char *>(key_.data()), iv, (is_encrypt ? 1 : 0));
+        RUNTIME_CHECK_MSG(ret == 1, "Failed to create cipher context.");
+
+        // Disable padding. After disabling padding, data size should always be
+        // multiply of block size.
+        ret = EVP_CIPHER_CTX_set_padding(ctx, 0);
+        RUNTIME_CHECK_MSG(ret == 1, "Failed to disable padding for cipher context.");
+    }
+
+    // In the following we assume the encrypt/decrypt process allow in and out buffer are
     // the same, to save one memcpy. This is not specified in official man page.
 
     // Handle partial block at the beginning. The partial block is copied to
@@ -130,12 +117,26 @@ void AESCTRCipherStream::cipher(uint64_t file_offset, char * data, size_t data_s
     {
         size_t partial_block_size = std::min<size_t>(block_size - block_offset, remaining_data_size);
         memcpy(partial_block + block_offset, data, partial_block_size);
-        ret = EVP_CipherUpdate(ctx, partial_block, &output_size, partial_block, block_size);
-        RUNTIME_CHECK_MSG(ret == 1, "Cipher failed for first block, offset {}.", file_offset);
-        RUNTIME_CHECK_MSG(output_size == static_cast<int>(block_size),
-                          "Unexpected cipher output size for first block, expected {} actual {}",
-                          block_size,
-                          output_size);
+#if USE_GM_SSL
+        if (cipher_ == nullptr)
+        {
+            if (is_encrypt)
+                sm4_ctr_encrypt(&sm4_key_, iv, partial_block, block_size, partial_block);
+            else
+                sm4_ctr_decrypt(&sm4_key_, iv, partial_block, block_size, partial_block);
+        }
+        else
+        {
+#endif
+            ret = EVP_CipherUpdate(ctx, partial_block, &output_size, partial_block, block_size);
+            RUNTIME_CHECK_MSG(ret == 1, "Cipher failed for first block, offset {}.", file_offset);
+            RUNTIME_CHECK_MSG(output_size == static_cast<int>(block_size),
+                              "Unexpected cipher output size for first block, expected {} actual {}",
+                              block_size,
+                              output_size);
+#if USE_GM_SSL
+        }
+#endif
         memcpy(data, partial_block + block_offset, partial_block_size);
         data_offset += partial_block_size;
         remaining_data_size -= partial_block_size;
@@ -146,12 +147,30 @@ void AESCTRCipherStream::cipher(uint64_t file_offset, char * data, size_t data_s
     {
         size_t actual_data_size = remaining_data_size - remaining_data_size % block_size;
         unsigned char * full_blocks = reinterpret_cast<unsigned char *>(data) + data_offset;
-        ret = EVP_CipherUpdate(ctx, full_blocks, &output_size, full_blocks, static_cast<int>(actual_data_size));
-        RUNTIME_CHECK_MSG(ret == 1, "Cipher failed for offset {}.", file_offset + data_offset);
-        RUNTIME_CHECK_MSG(output_size == static_cast<int>(actual_data_size),
-                          "Unexpected cipher output size for block, expected {} actual {}",
-                          actual_data_size,
-                          output_size);
+#if USE_GM_SSL
+        if (cipher_ == nullptr)
+        {
+            if (is_encrypt)
+            {
+                sm4_ctr_encrypt(&sm4_key_, iv, full_blocks, actual_data_size, full_blocks);
+            }
+            else
+            {
+                sm4_ctr_decrypt(&sm4_key_, iv, full_blocks, actual_data_size, full_blocks);
+            }
+        }
+        else
+        {
+#endif
+            ret = EVP_CipherUpdate(ctx, full_blocks, &output_size, full_blocks, static_cast<int>(actual_data_size));
+            RUNTIME_CHECK_MSG(ret == 1, "Cipher failed for offset {}.", file_offset + data_offset);
+            RUNTIME_CHECK_MSG(output_size == static_cast<int>(actual_data_size),
+                              "Unexpected cipher output size for block, expected {} actual {}",
+                              actual_data_size,
+                              output_size);
+#if USE_GM_SSL
+        }
+#endif
         data_offset += actual_data_size;
         remaining_data_size -= actual_data_size;
     }
@@ -162,17 +181,51 @@ void AESCTRCipherStream::cipher(uint64_t file_offset, char * data, size_t data_s
     {
         assert(remaining_data_size < AES_BLOCK_SIZE);
         memcpy(partial_block, data + data_offset, remaining_data_size);
-        ret = EVP_CipherUpdate(ctx, partial_block, &output_size, partial_block, block_size);
-        RUNTIME_CHECK_MSG(ret == 1, "Cipher failed for last block, offset {}.", file_offset + data_offset);
-        RUNTIME_CHECK_MSG(output_size == static_cast<int>(block_size),
-                          "Unexpected cipher output size for last block, expected {} actual {}",
-                          block_size,
-                          output_size);
+#if USE_GM_SSL
+        if (cipher_ == nullptr)
+        {
+            if (is_encrypt)
+            {
+                sm4_ctr_encrypt(&sm4_key_, iv, partial_block, block_size, partial_block);
+            }
+            else
+            {
+                sm4_ctr_decrypt(&sm4_key_, iv, partial_block, block_size, partial_block);
+            }
+        }
+        else
+        {
+#endif
+            ret = EVP_CipherUpdate(ctx, partial_block, &output_size, partial_block, block_size);
+            RUNTIME_CHECK_MSG(ret == 1, "Cipher failed for last block, offset {}.", file_offset + data_offset);
+            RUNTIME_CHECK_MSG(output_size == static_cast<int>(block_size),
+                              "Unexpected cipher output size for last block, expected {} actual {}",
+                              block_size,
+                              output_size);
+#if USE_GM_SSL
+        }
+#endif
         memcpy(data + data_offset, partial_block, remaining_data_size);
     }
 #endif
 }
 
+inline void AESCTRCipherStream::initIV(uint64_t block_index, unsigned char * iv) const
+{
+    // In CTR mode, OpenSSL EVP API treat the IV as a 128-bit big-endian, and
+    // increase it by 1 for each block.
+    uint64_t iv_high = initial_iv_high_;
+    uint64_t iv_low = initial_iv_low_ + block_index;
+    if (std::numeric_limits<uint64_t>::max() - block_index < initial_iv_low_)
+    {
+        iv_high++;
+    }
+    iv_high = toBigEndian(iv_high);
+    iv_low = toBigEndian(iv_low);
+    memcpy(iv, &iv_high, sizeof(uint64_t));
+    memcpy(iv + sizeof(uint64_t), &iv_low, sizeof(uint64_t));
+}
+
 BlockAccessCipherStreamPtr AESCTRCipherStream::createCipherStream(
     const FileEncryptionInfo & encryption_info_,
     const EncryptionPath & encryption_path_)
@@ -195,7 +248,10 @@ BlockAccessCipherStreamPtr AESCTRCipherStream::createCipherStream(
         cipher = EVP_aes_256_ctr();
         break;
     case EncryptionMethod::SM4Ctr:
-#if OPENSSL_VERSION_NUMBER < 0x1010100fL || defined(OPENSSL_NO_SM4)
+#if USE_GM_SSL
+        // Use sm4 in GmSSL, don't need to do anything here
+        break;
+#elif OPENSSL_VERSION_NUMBER < 0x1010100fL || defined(OPENSSL_NO_SM4)
         throw DB::TiFlashException("Unsupported encryption method: " + std::to_string(static_cast<int>(encryption_info_.method)),
                                    Errors::Encryption::Internal);
 #else