From 8a4670becd8ee1c8365a606f8cc90edd784cd0aa Mon Sep 17 00:00:00 2001 From: Jeremy Mikola Date: Mon, 6 Dec 2021 11:15:20 -0500 Subject: [PATCH] PHPC-2011: Upgrade libmongoc to 1.20.0 and libmongocrypt to 1.3.0 (#1279) * PHPC-2011: Bump libmongocrypt to 1.3.0 * PHPC-2011: Bump libmongoc to 1.20.0 * Script to update submodule sources in config.m4 and config.w32 --- CONTRIBUTING.md | 22 ++++++----- config.m4 | 5 ++- config.w32 | 5 ++- scripts/autotools/libmongocrypt/Endian.m4 | 6 +++ scripts/update-submodule-sources.php | 47 +++++++++++++++++++++++ src/LIBMONGOCRYPT_VERSION_CURRENT | 2 +- src/LIBMONGOC_VERSION_CURRENT | 2 +- src/libmongoc | 2 +- src/libmongocrypt | 2 +- 9 files changed, 75 insertions(+), 18 deletions(-) create mode 100644 scripts/autotools/libmongocrypt/Endian.m4 create mode 100644 scripts/update-submodule-sources.php diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fa3fcfff..825f4234 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -136,7 +136,7 @@ reflect the new sources and/or package version. ``` $ cd src/libmongoc $ git fetch -$ git checkout 1.15.0 +$ git checkout 1.20.0 ``` During development, it may be necessary to temporarily point the libmongoc @@ -175,9 +175,10 @@ collectively enumerate all of the the sources within the libmongoc submodule to include in a bundled build. These variables should each have a shell command in a preceding comment, which -should be run to regenerate that particular list of source files. In the event -that either libmongoc or libbson introduce a new source directory, that will -need to be manually added (follow prior art). +should be run to regenerate that particular list of source files. Each command +may be run manually or `scripts/update-submodule-sources.php` may be used to +update all variables. In the event that either libmongoc or libbson introduce a +new source directory, that will need to be manually added (follow prior art). #### Update package dependencies @@ -189,11 +190,11 @@ error message in the `pkg-config` blocks for both libmongoc and libbson. For example, the following lines might be updated for libmongoc: ``` -if $PKG_CONFIG libmongoc-1.0 --atleast-version 1.15.0; then +if $PKG_CONFIG libmongoc-1.0 --atleast-version 1.20.0; then ... -AC_MSG_ERROR(system libmongoc must be upgraded to version >= 1.15.0) +AC_MSG_ERROR(system libmongoc must be upgraded to version >= 1.20.0) ``` #### Update tested versions in Evergreen configuration @@ -221,7 +222,7 @@ and by ensuring that the test suite passes. Once done, commit the changes to all of the above files/paths. For example: ``` -$ git commit -m "Bump libmongoc to 1.15.0" config.m4 config.w32 src/libmongoc src/LIBMONGOC_VERSION_CURRENT +$ git commit -m "Bump libmongoc to 1.20.0" config.m4 config.w32 src/libmongoc src/LIBMONGOC_VERSION_CURRENT ``` ### Updating libmongocrypt @@ -231,12 +232,13 @@ To update libmongocrypt, the steps are similar to the above: ``` $ cd src/libmongocrypt $ git fetch -$ git checkout 1.0.1 +$ git checkout 1.3.0 $ make libmongocrypt-version-current ``` -Package dependencies in `config.m4` must also be updated, as do the sources in -the PECL generation script. +Package dependencies in `config.m4` must also be updated (either manually or +with `scripts/update-submodule-sources.php`), as do the sources in the PECL +generation script. ## Releasing diff --git a/config.m4 b/config.m4 index 1b6239de..5b4dafe3 100644 --- a/config.m4 +++ b/config.m4 @@ -338,6 +338,7 @@ if test "$PHP_MONGODB" != "no"; then dnl This include modifies the value of $PHP_MONGODB_CLIENT_SIDE_ENCRYPTION to "yes" dnl or "no" depending on whether dependencies for libmongocrypt are fulfilled _include([scripts/autotools/libmongocrypt/CheckSSL.m4]) + _include([scripts/autotools/libmongocrypt/Endian.m4]) _include([scripts/autotools/libmongocrypt/Version.m4]) PHP_MONGODB_BSON_VERSION_STRING="Bundled ($BSON_VERSION)" @@ -443,7 +444,7 @@ if test "$PHP_MONGODB" != "no"; then AC_SUBST(MONGOCRYPT_ENABLE_TRACE, 1) dnl Generated with: find src/libmongocrypt/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 4- | sort -dz | tr '\000' ' ' - PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c" + PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c mongocrypt-util.c" dnl Generated with: find src/libmongocrypt/src/crypto -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' ' PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES="cng.c commoncrypto.c libcrypto.c none.c" @@ -455,7 +456,7 @@ if test "$PHP_MONGODB" != "no"; then PHP_MONGODB_MONGOCRYPT_OS_WIN_SOURCES="os_mutex.c os_once.c" dnl Generated with: find src/libmongocrypt/kms-message/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' ' - PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c" + PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kmip_reader_writer.c kms_kmip_request.c kms_kmip_response.c kms_kmip_response_parser.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c" PHP_MONGODB_LIBMONGOCRYPT_CFLAGS="$PHP_MONGODB_BUNDLED_CFLAGS $PHP_MONGODB_LIBMONGOCRYPT_CFLAGS" PHP_MONGODB_ADD_SOURCES([src/libmongocrypt/src/], $PHP_MONGODB_MONGOCRYPT_SOURCES, $PHP_MONGODB_LIBMONGOCRYPT_CFLAGS) diff --git a/config.w32 b/config.w32 index 1918e0e4..82492909 100644 --- a/config.w32 +++ b/config.w32 @@ -264,6 +264,7 @@ if (PHP_MONGODB != "no") { if (PHP_MONGODB_CLIENT_SIDE_ENCRYPTION != "no" && mongoc_ssl_found) { mongoc_opts.MONGOC_ENABLE_CLIENT_SIDE_ENCRYPTION = 1; + ADD_FLAG("CFLAGS_MONGODB", "/D KMS_MESSAGE_LITTLE_ENDIAN=1"); ADD_FLAG("CFLAGS_MONGODB", "/I" + configure_module_dirname + "/src/libmongocrypt/src"); ADD_FLAG("CFLAGS_MONGODB", "/I" + configure_module_dirname + "/src/libmongocrypt/kms-message/src"); ADD_FLAG("CFLAGS_MONGODB", "/I" + configure_module_dirname + "/src/libmongocrypt-compat"); @@ -279,7 +280,7 @@ if (PHP_MONGODB != "no") { }; // Generated with: find src/libmongocrypt/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 4- | sort -dz | tr '\000' ' ' - var PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c"; + var PHP_MONGODB_MONGOCRYPT_SOURCES="mongocrypt-binary.c mongocrypt-buffer.c mongocrypt.c mongocrypt-cache.c mongocrypt-cache-collinfo.c mongocrypt-cache-key.c mongocrypt-cache-oauth.c mongocrypt-ciphertext.c mongocrypt-crypto.c mongocrypt-ctx.c mongocrypt-ctx-datakey.c mongocrypt-ctx-decrypt.c mongocrypt-ctx-encrypt.c mongocrypt-endpoint.c mongocrypt-kek.c mongocrypt-key-broker.c mongocrypt-key.c mongocrypt-kms-ctx.c mongocrypt-log.c mongocrypt-marking.c mongocrypt-opts.c mongocrypt-status.c mongocrypt-traverse-util.c mongocrypt-util.c"; // Generated with: find src/libmongocrypt/src/crypto -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' ' var PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES="cng.c commoncrypto.c libcrypto.c none.c"; @@ -291,7 +292,7 @@ if (PHP_MONGODB != "no") { var PHP_MONGODB_MONGOCRYPT_OS_WIN_SOURCES="os_mutex.c os_once.c"; // Generated with: find src/libmongocrypt/kms-message/src -maxdepth 1 -name '*.c' -print0 | cut -sz -d / -f 5- | sort -dz | tr '\000' ' ' - var PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c"; + var PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES="hexlify.c kms_azure_request.c kms_b64.c kms_caller_identity_request.c kms_crypto_apple.c kms_crypto_libcrypto.c kms_crypto_none.c kms_crypto_windows.c kms_decrypt_request.c kms_encrypt_request.c kms_gcp_request.c kms_kmip_reader_writer.c kms_kmip_request.c kms_kmip_response.c kms_kmip_response_parser.c kms_kv_list.c kms_message.c kms_port.c kms_request.c kms_request_opt.c kms_request_str.c kms_response.c kms_response_parser.c sort.c"; MONGODB_ADD_SOURCES("/src/libmongocrypt/src", PHP_MONGODB_MONGOCRYPT_SOURCES); MONGODB_ADD_SOURCES("/src/libmongocrypt/src/crypto", PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES); diff --git a/scripts/autotools/libmongocrypt/Endian.m4 b/scripts/autotools/libmongocrypt/Endian.m4 new file mode 100644 index 00000000..aac4a5b6 --- /dev/null +++ b/scripts/autotools/libmongocrypt/Endian.m4 @@ -0,0 +1,6 @@ +AC_C_BIGENDIAN +if test "x$ac_cv_c_bigendian" = "xyes"; then + PHP_MONGODB_BUNDLED_CFLAGS="$PHP_MONGODB_BUNDLED_CFLAGS -DKMS_MESSAGE_BIG_ENDIAN=1" +else + PHP_MONGODB_BUNDLED_CFLAGS="$PHP_MONGODB_BUNDLED_CFLAGS -DKMS_MESSAGE_LITTLE_ENDIAN=1" +fi diff --git a/scripts/update-submodule-sources.php b/scripts/update-submodule-sources.php new file mode 100644 index 00000000..b526d337 --- /dev/null +++ b/scripts/update-submodule-sources.php @@ -0,0 +1,47 @@ + 'src/libmongoc/src/common', + 'PHP_MONGODB_KMS_MESSAGE_SOURCES' => 'src/libmongoc/src/kms-message/src', + 'PHP_MONGODB_BSON_SOURCES' => 'src/libmongoc/src/libbson/src/bson', + 'PHP_MONGODB_JSONSL_SOURCES' => 'src/libmongoc/src/libbson/src/jsonsl', + 'PHP_MONGODB_MONGOC_SOURCES' => 'src/libmongoc/src/libmongoc/src/mongoc', + 'PHP_MONGODB_ZLIB_SOURCES' => 'src/libmongoc/src/zlib-1.2.11', + 'PHP_MONGODB_MONGOCRYPT_SOURCES' => 'src/libmongocrypt/src', + 'PHP_MONGODB_MONGOCRYPT_CRYPTO_SOURCES' => 'src/libmongocrypt/src/crypto', + 'PHP_MONGODB_MONGOCRYPT_OS_POSIX_SOURCES' => 'src/libmongocrypt/src/os_posix', + 'PHP_MONGODB_MONGOCRYPT_OS_WIN_SOURCES' => 'src/libmongocrypt/src/os_win', + 'PHP_MONGODB_MONGOCRYPT_KMS_MESSAGE_SOURCES' => 'src/libmongocrypt/kms-message/src', +]; + +$patterns = []; +$replacements = []; + +foreach ($vars as $var => $path) { + $cutNth = 2 + substr_count($path, '/'); + + $files = trim(shell_exec(sprintf($cmd, $path, $cutNth))); + + $patterns[] = sprintf('/(%s=")([^"]*)(";?)/', $var); + $replacements[] = '$1' . $files . '$3'; +} + +$files = [ + realpath(__DIR__ . '/../config.m4') => count($patterns), + // config.w32 does not use PHP_MONGODB_ZLIB_SOURCES (PHPC-1111) + realpath(__DIR__ . '/../config.w32') => count($patterns) - 1, +]; + +foreach ($files as $file => $expectedCount) { + $replaced = preg_replace($patterns, $replacements, file_get_contents($file), 1, $count); + + if ($count !== $expectedCount) { + fprintf(STDERR, "Skipping %s: Expected %d replacements but only matched %d\n", basename($file), $expectedCount, $count); + continue; + } + + printf("Updated %s\n", basename($file)); + file_put_contents($file, $replaced); +} diff --git a/src/LIBMONGOCRYPT_VERSION_CURRENT b/src/LIBMONGOCRYPT_VERSION_CURRENT index 6085e946..f0bb29e7 100644 --- a/src/LIBMONGOCRYPT_VERSION_CURRENT +++ b/src/LIBMONGOCRYPT_VERSION_CURRENT @@ -1 +1 @@ -1.2.1 +1.3.0 diff --git a/src/LIBMONGOC_VERSION_CURRENT b/src/LIBMONGOC_VERSION_CURRENT index e83e063d..39893559 100644 --- a/src/LIBMONGOC_VERSION_CURRENT +++ b/src/LIBMONGOC_VERSION_CURRENT @@ -1 +1 @@ -1.11.1-20211103+gitb93e345178 +1.20.0 diff --git a/src/libmongoc b/src/libmongoc index b93e3451..03be6bff 160000 --- a/src/libmongoc +++ b/src/libmongoc @@ -1 +1 @@ -Subproject commit b93e345178a3a50352894b4be300928987f3d371 +Subproject commit 03be6bffd11503958150d2dfe207f3bcdc2638cb diff --git a/src/libmongocrypt b/src/libmongocrypt index 811df516..53068ae7 160000 --- a/src/libmongocrypt +++ b/src/libmongocrypt @@ -1 +1 @@ -Subproject commit 811df516e2985a08eab5b5e0eeca00823b2a2a58 +Subproject commit 53068ae7101644b8bd16ecd13439a2dd3e7dd617