From 164c4f808b9606a771d65f8c575f176f737cf18b Mon Sep 17 00:00:00 2001 From: Leonard Grey Date: Tue, 30 Sep 2025 19:50:37 -0400 Subject: [PATCH 1/3] Build Brotli in advance of it being a Foundation dep Also builds cURL with Brotli to match https://github.com/swiftlang/swift/pull/83441/files --- .github/workflows/build-toolchain.yml | 8 ++ .github/workflows/swift-toolchain.yml | 107 +++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-toolchain.yml b/.github/workflows/build-toolchain.yml index 8eaf7bf02..3793b3648 100644 --- a/.github/workflows/build-toolchain.yml +++ b/.github/workflows/build-toolchain.yml @@ -156,6 +156,8 @@ jobs: context: runs-on: ubuntu-latest outputs: + brotli_revision: ${{ steps.context.outputs.brotli_revision }} + brotli_version: ${{ steps.context.outputs.brotli_version }} curl_revision: ${{ steps.context.outputs.curl_revision }} curl_version: ${{ steps.context.outputs.curl_version }} ds2_revision: ${{ steps.context.outputs.ds2_revision }} @@ -281,6 +283,7 @@ jobs: ds2_revision=refs/tags/nightly-2024-11-07 libxml2_revision=refs/tags/v2.11.5 zlib_revision=refs/tags/v1.3.1 + brotli_revision=refs/tags/v1.1.0 EOF else repo manifest -r --suppress-upstream-revision --suppress-dest-branch | \ @@ -299,6 +302,7 @@ jobs: echo swift_toolchain_sqlite_version=3.46.0 >> ${GITHUB_OUTPUT} echo swift_cmark_version=0.29.0.gfm.13 >> ${GITHUB_OUTPUT} echo zlib_version=1.3 >> ${GITHUB_OUTPUT} + echo brotli_version=1.1.0 >> ${GITHUB_OUTPUT} # FIXME(z2oh): Remove /D_DISABLE_CONSTEXPR_MUTEX_CONSTRUCTOR when GitHub runner image updates to 20240610.1. # see: https://github.com/actions/runner-images/issues/10004 @@ -569,6 +573,8 @@ jobs: curl_revision: ${{ needs.context.outputs.curl_revision }} curl_version: ${{ needs.context.outputs.curl_version }} ds2_revision: ${{ needs.context.outputs.ds2_revision }} + brotli_revision: ${{ needs.context.outputs.brotli_revision }} + brotli_version: ${{ needs.context.outputs.brotli_version }} indexstore_db_revision: ${{ needs.context.outputs.indexstore_db_revision }} libxml2_revision: ${{ needs.context.outputs.libxml2_revision }} libxml2_version: ${{ needs.context.outputs.libxml2_version }} @@ -648,6 +654,8 @@ jobs: curl_revision: ${{ needs.context.outputs.curl_revision }} curl_version: ${{ needs.context.outputs.curl_version }} ds2_revision: ${{ needs.context.outputs.ds2_revision }} + brotli_revision: ${{ needs.context.outputs.brotli_revision }} + brotli_version: ${{ needs.context.outputs.brotli_version }} indexstore_db_revision: ${{ needs.context.outputs.indexstore_db_revision }} libxml2_revision: ${{ needs.context.outputs.libxml2_revision }} libxml2_version: ${{ needs.context.outputs.libxml2_version }} diff --git a/.github/workflows/swift-toolchain.yml b/.github/workflows/swift-toolchain.yml index deb29a4ef..423f37b66 100644 --- a/.github/workflows/swift-toolchain.yml +++ b/.github/workflows/swift-toolchain.yml @@ -191,6 +191,14 @@ on: required: true type: string + brotli_revision: + required: true + type: string + + brotli_version: + required: true + type: string + ANDROID_API_LEVEL: required: true type: string @@ -1311,8 +1319,96 @@ jobs: name: ${{ matrix.os }}-${{ matrix.arch }}-zlib-${{ inputs.zlib_version }} path: ${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr + brotli: + runs-on: ${{ inputs.default_build_runner }} + + strategy: + fail-fast: false + matrix: ${{ fromJSON(inputs.target_matrix) }} + + name: ${{ matrix.os }} ${{ matrix.arch }} brotli + + steps: + - uses: actions/checkout@v4.2.2 + with: + path: ${{ github.workspace }}/SourceCache/ci-build + show-progress: false + + - uses: ./SourceCache/ci-build/.github/actions/setup-build + with: + setup-vs-dev-env: ${{ matrix.os == 'Windows' }} + host-arch: ${{ matrix.arch }} + - uses: seanmiddleditch/gha-setup-ninja@96bed6edff20d1dd61ecff9b75cc519d516e6401 # master + if: inputs.build_os == 'Darwin' + + - name: Compute workspace hash + id: workspace_hash + run: | + $stringAsStream = [System.IO.MemoryStream]::new() + $writer = [System.IO.StreamWriter]::new($stringAsStream) + $writer.write("${{ github.workspace }}") + $writer.Flush() + $stringAsStream.Position = 0 + $hash = (Get-FileHash -Algorithm SHA256 -InputStream $stringAsStream).Hash + echo "hash=$hash" >> $env:GITHUB_OUTPUT + - name: Setup sccache + uses: ./SourceCache/ci-build/.github/actions/setup-sccache + with: + s3-bucket: ${{ vars.SCCACHE_S3_BUCKET }} + aws-region: ${{ vars.SCCACHE_AWS_REGION }} + aws-arn: ${{ vars.SCCACHE_AWS_ARN }} + disk-max-size: 100M + disk-cache-key: ${{ steps.workspace_hash.outputs.hash }}-${{ matrix.os }}-${{ matrix.arch }}-brotli + + - uses: actions/checkout@v4.2.2 + with: + repository: google/brotli + ref: ${{ inputs.brotli_revision }} + path: ${{ github.workspace }}/SourceCache/brotli + show-progress: false + + - uses: nttld/setup-ndk@v1 + if: matrix.os == 'Android' + id: setup-ndk + with: + ndk-version: ${{ inputs.ANDROID_NDK_VERSION }} + local-cache: true + + - name: Configure brotli + uses: ./SourceCache/ci-build/.github/actions/configure-cmake-project + with: + project-name: brotli + swift-version: ${{ inputs.swift_version }} + enable-caching: true + debug-info: ${{ inputs.debug_info }} + build-os: ${{ inputs.build_os }} + build-arch: ${{ inputs.build_arch }} + os: ${{ matrix.os }} + arch: ${{ matrix.arch }} + src-dir: ${{ github.workspace }}/SourceCache/brotli + bin-dir: ${{ github.workspace }}/BinaryCache/brotli-${{ inputs.brotli_version }} + install-dir: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr + android-api-level: ${{ inputs.ANDROID_API_LEVEL }} + android-clang-version: ${{ inputs.ANDROID_CLANG_VERSION }} + ndk-path: ${{ steps.setup-ndk.outputs.ndk-path }} + msvc-compilers: '@("C")' + cmake-defines: | + @{ + 'BUILD_SHARED_LIBS' = "NO"; + 'CMAKE_POSITION_INDEPENDENT_CODE' = "YES"; + } + - name: Build brotli + run: cmake --build ${{ github.workspace }}/BinaryCache/brotli-${{ inputs.brotli_version }} + - name: Install brotli + run: cmake --build ${{ github.workspace }}/BinaryCache/brotli-${{ inputs.brotli_version }} --target install + + - uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.arch }}-brotli-${{ inputs.brotli_version }} + path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr + curl: - needs: [zlib] + needs: [zlib, brotli] runs-on: ${{ inputs.default_build_runner }} strategy: @@ -1364,6 +1460,10 @@ jobs: with: name: ${{ matrix.os }}-${{ matrix.arch }}-zlib-${{ inputs.zlib_version }} path: ${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr + - uses: actions/download-artifact@v4 + with: + name: ${{ matrix.os }}-${{ matrix.arch }}-brotli-${{ inputs.brotli_version }} + path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr - uses: nttld/setup-ndk@v1 if: matrix.os == 'Android' @@ -1401,7 +1501,7 @@ jobs: 'CURL_CA_BUNDLE' = "none"; 'CURL_CA_FALLBACK' = "NO"; 'CURL_CA_PATH' = "none"; - 'CURL_BROTLI' = "NO"; + 'CURL_BROTLI' = "YES"; 'CURL_DISABLE_ALTSVC' = "NO"; 'CURL_DISABLE_AWS' = "YES"; 'CURL_DISABLE_BASIC_AUTH' = "NO"; @@ -1479,6 +1579,9 @@ jobs: 'USE_WIN32_LDAP' = "NO"; 'ZLIB_ROOT' = "${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr"; 'ZLIB_LIBRARY' = "${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr/lib/zlibstatic.lib"; + 'BROTLI_INCLUDE_DIR' = "${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr/include"; + 'BROTLICOMMON_LIBRARY' = "${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr/lib/brotlicommon.lib"; + 'BROTLIDEC_LIBRARY' = "${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr/lib/brotlidec.lib"; } - name: Build curl run: cmake --build ${{ github.workspace }}/BinaryCache/curl-${{ inputs.curl_version }} From cb484e92691e0b63d38c131ad14a641b9bc63736 Mon Sep 17 00:00:00 2001 From: Leonard Grey Date: Thu, 2 Oct 2025 11:20:03 -0400 Subject: [PATCH 2/3] Download brotli artifact in SDK jobs --- .github/workflows/swift-toolchain.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/swift-toolchain.yml b/.github/workflows/swift-toolchain.yml index 423f37b66..78569f570 100644 --- a/.github/workflows/swift-toolchain.yml +++ b/.github/workflows/swift-toolchain.yml @@ -2140,7 +2140,12 @@ jobs: with: name: ${{ matrix.os }}-${{ matrix.arch }}-zlib-${{ inputs.zlib_version }} path: ${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr - + - uses: actions/download-artifact@v4 + if: matrix.os != 'Android' || inputs.build_android + with: + name: ${{ matrix.os }}-${{ matrix.arch }}-brotli-${{ inputs.brotli_version }} + path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr + - name: Download Compilers if: matrix.os != 'Android' || inputs.build_android uses: thebrowsercompany/gha-download-tar-artifact@59992d91335d4ecba543c8535f7d07238e42125d # main @@ -2801,7 +2806,12 @@ jobs: with: name: ${{ matrix.os }}-${{ matrix.arch }}-zlib-${{ inputs.zlib_version }} path: ${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr - + - uses: actions/download-artifact@v4 + if: matrix.os != 'Android' || inputs.build_android + with: + name: ${{ matrix.os }}-${{ matrix.arch }}-brotli-${{ inputs.brotli_version }} + path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr + - name: Download Compilers if: matrix.os != 'Android' || inputs.build_android uses: thebrowsercompany/gha-download-tar-artifact@59992d91335d4ecba543c8535f7d07238e42125d # main From 4342e1963b4567091f545364c3fbee693d7acb19 Mon Sep 17 00:00:00 2001 From: Leonard Grey Date: Thu, 2 Oct 2025 14:09:37 -0400 Subject: [PATCH 3/3] Just build, no cURL --- .github/workflows/swift-toolchain.yml | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/.github/workflows/swift-toolchain.yml b/.github/workflows/swift-toolchain.yml index 78569f570..feb0c72de 100644 --- a/.github/workflows/swift-toolchain.yml +++ b/.github/workflows/swift-toolchain.yml @@ -1408,7 +1408,7 @@ jobs: path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr curl: - needs: [zlib, brotli] + needs: [zlib] runs-on: ${{ inputs.default_build_runner }} strategy: @@ -1460,10 +1460,6 @@ jobs: with: name: ${{ matrix.os }}-${{ matrix.arch }}-zlib-${{ inputs.zlib_version }} path: ${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr - - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.os }}-${{ matrix.arch }}-brotli-${{ inputs.brotli_version }} - path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr - uses: nttld/setup-ndk@v1 if: matrix.os == 'Android' @@ -1501,7 +1497,7 @@ jobs: 'CURL_CA_BUNDLE' = "none"; 'CURL_CA_FALLBACK' = "NO"; 'CURL_CA_PATH' = "none"; - 'CURL_BROTLI' = "YES"; + 'CURL_BROTLI' = "NO"; 'CURL_DISABLE_ALTSVC' = "NO"; 'CURL_DISABLE_AWS' = "YES"; 'CURL_DISABLE_BASIC_AUTH' = "NO"; @@ -1579,9 +1575,6 @@ jobs: 'USE_WIN32_LDAP' = "NO"; 'ZLIB_ROOT' = "${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr"; 'ZLIB_LIBRARY' = "${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr/lib/zlibstatic.lib"; - 'BROTLI_INCLUDE_DIR' = "${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr/include"; - 'BROTLICOMMON_LIBRARY' = "${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr/lib/brotlicommon.lib"; - 'BROTLIDEC_LIBRARY' = "${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr/lib/brotlidec.lib"; } - name: Build curl run: cmake --build ${{ github.workspace }}/BinaryCache/curl-${{ inputs.curl_version }} @@ -2140,12 +2133,7 @@ jobs: with: name: ${{ matrix.os }}-${{ matrix.arch }}-zlib-${{ inputs.zlib_version }} path: ${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr - - uses: actions/download-artifact@v4 - if: matrix.os != 'Android' || inputs.build_android - with: - name: ${{ matrix.os }}-${{ matrix.arch }}-brotli-${{ inputs.brotli_version }} - path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr - + - name: Download Compilers if: matrix.os != 'Android' || inputs.build_android uses: thebrowsercompany/gha-download-tar-artifact@59992d91335d4ecba543c8535f7d07238e42125d # main @@ -2806,12 +2794,7 @@ jobs: with: name: ${{ matrix.os }}-${{ matrix.arch }}-zlib-${{ inputs.zlib_version }} path: ${{ github.workspace }}/BuildRoot/Library/zlib-${{ inputs.zlib_version }}/usr - - uses: actions/download-artifact@v4 - if: matrix.os != 'Android' || inputs.build_android - with: - name: ${{ matrix.os }}-${{ matrix.arch }}-brotli-${{ inputs.brotli_version }} - path: ${{ github.workspace }}/BuildRoot/Library/brotli-${{ inputs.brotli_version }}/usr - + - name: Download Compilers if: matrix.os != 'Android' || inputs.build_android uses: thebrowsercompany/gha-download-tar-artifact@59992d91335d4ecba543c8535f7d07238e42125d # main