diff --git a/.azure-pipelines/release.yml b/.azure-pipelines/release.yml index f0e79b31926149..d5d34050862f39 100644 --- a/.azure-pipelines/release.yml +++ b/.azure-pipelines/release.yml @@ -35,6 +35,9 @@ parameters: os: windows toolchain: x86_64 mingwprefix: mingw64 + # GitHub repo whose ci-artifacts release exposes a + # build-installers.tar.zst snapshot of the matching SDK. + sdk_repo: git-for-windows/git-sdk-64 - id: windows_arm64 jobName: 'Windows (ARM64)' @@ -44,6 +47,7 @@ parameters: os: windows toolchain: clang-aarch64 mingwprefix: clangarm64 + sdk_repo: git-for-windows/git-sdk-arm64 - name: macos_matrix type: object @@ -150,6 +154,7 @@ extends: git_version: $[stageDependencies.prereqs.prebuild.outputs['info.git_version']] toolchain: ${{ dim.toolchain }} mingwprefix: ${{ dim.mingwprefix }} + sdk_repo: ${{ dim.sdk_repo }} templateContext: outputs: - output: pipelineArtifact @@ -175,6 +180,134 @@ extends: Write-Host "##vso[task.prependpath]$azPath" Write-Host "Azure CLI installed." displayName: 'Install Azure CLI (arm64)' + - task: PowerShell@2 + displayName: 'Bootstrap Git for Windows SDK' + inputs: + targetType: inline + script: | + $ErrorActionPreference = 'Stop' + $ProgressPreference = 'SilentlyContinue' + + # Win 2022's built-in tar.exe (BSD tar) does not + # decompress zstd archives, and the agent image is + # not guaranteed to ship Git for Windows. Download + # tar.exe + zstd.exe + msys-zstd-1.dll straight + # from git-sdk-64's HEAD and use those. + $tools = "$(Build.SourcesDirectory)\sdk-tools" + New-Item -ItemType Directory -Path $tools -Force | Out-Null + $base = 'https://github.com/git-for-windows/git-sdk-64/raw/HEAD/usr/bin' + foreach ($f in 'tar.exe', 'zstd.exe', 'msys-zstd-1.dll') { + Invoke-WebRequest "$base/$f" -OutFile "$tools\$f" + } + + # Make tar/zstd findable inside this task too; + # ##vso[task.prependpath] only affects subsequent + # tasks. + $env:PATH = "$tools;$env:PATH" + + # Download the SDK snapshot for the matching arch + # from its ci-artifacts rolling release. + $archive = "$(Build.SourcesDirectory)\build-installers.tar.zst" + Invoke-WebRequest ` + "https://github.com/$(sdk_repo)/releases/download/ci-artifacts/build-installers.tar.zst" ` + -OutFile $archive + + # Extract the SDK into C:\sdk. + $sdk = 'C:\sdk' + New-Item -ItemType Directory -Path $sdk -Force | Out-Null + & tar.exe --use-compress-program='zstd -d' -xf $archive -C $sdk + + # Make tar tools, the SDK's bash, and the matching + # MinGW toolchain available to subsequent tasks. + Write-Host "##vso[task.prependpath]$tools" + Write-Host "##vso[task.prependpath]$sdk\usr\bin" + Write-Host "##vso[task.prependpath]$sdk\$(mingwprefix)\bin" + - task: Bash@3 + displayName: 'Clone build-extra into SDK' + inputs: + targetType: inline + script: | + set -euo pipefail + # The please.sh + signtool.sh scripts the build + # relies on live in build-extra; the SDK ships + # without them. Partial clone to keep this fast. + git clone --filter=blob:none --single-branch -b main \ + https://github.com/git-for-windows/build-extra \ + /usr/src/build-extra + - task: Bash@3 + displayName: 'Build mingw-w64-git package' + inputs: + targetType: inline + script: | + set -euo pipefail + set -x + + # makepkg-mingw runs git internally and looks for a + # plain /usr/bin/git; trampoline it to the matching + # MinGW-built git.exe. + printf '#!/bin/sh\n\nexec /%s/bin/git.exe "$@"\n' \ + "$(mingwprefix)" >/usr/bin/git + chmod +x /usr/bin/git + + USER_NAME='microsoft-git-build' + USER_EMAIL='microsoft-git-build@users.noreply.github.com' + git config --global user.name "$USER_NAME" + git config --global user.email "$USER_EMAIL" + export PACKAGER="$USER_NAME <$USER_EMAIL>" + + # please.sh's --only- takes the bare CPU + # name (x86_64 / aarch64), not the toolchain + # triple ($(toolchain) here is x86_64 or + # clang-aarch64). + case "$(toolchain)" in + x86_64) cpu_arch=x86_64 ;; + clang-aarch64) cpu_arch=aarch64 ;; + *) echo "unknown toolchain: $(toolchain)" >&2; exit 1 ;; + esac + + sh -x /usr/src/build-extra/please.sh build-mingw-w64-git \ + --only-"$cpu_arch" \ + --build-src-pkg \ + -o artifacts \ + HEAD + + # NOTE: the GitHub workflow additionally GPG-signs + # each tarball and creates a MINGW-packages.bundle + # for downstream Pacman consumers; both are + # intentionally out of scope for the initial port + # and tracked as follow-ups. + - task: Bash@3 + displayName: 'Build installer and portable Git' + inputs: + targetType: inline + script: | + set -euo pipefail + set -x + + b=/usr/src/build-extra + + # please.sh make_installers_from_mingw_w64_git + # --include-pdbs reads PDB archives from + # cached-source-packages/. + mkdir -p "$b/cached-source-packages" + cp artifacts/*-pdb* "$b/cached-source-packages/" + + # The --pkg=... list excludes the optional pieces + # the workflow drops (signatures, archimport, cvs, + # p4, gitweb, doc-man); keep the same filter so + # the resulting .exe size is comparable. + pkg_args=$( + ls artifacts/mingw-w64-$(toolchain)-*.tar.* \ + | sed '/\.sig$/d;/archimport/d;/cvs/d;/p4/d;/gitweb/d;/doc-man/d;s/^/--pkg=/' \ + | tr '\n' ' ' + ) + + for type in installer portable; do + eval "$b"/please.sh make_installers_from_mingw_w64_git --include-pdbs \ + --version="$(git_version)" \ + -o artifacts --"$type" \ + $pkg_args + done # Setup ESRP code signing for Windows (sets ESRP_TOOL, ESRP_AUTH) - ${{ if eq(parameters.esrp, true) }}: - template: .azure-pipelines/esrp/windows/setup.yml@self @@ -183,40 +316,37 @@ extends: esrpClientId: $(esrpClientId) keyVaultName: $(esrpKeyVaultName) signCertName: $(esrpSignReqCertName) - # TODO: add tasks to set up Git for Windows SDK - # TODO: add tasks to build Git and installers - - script: | - echo $(mingwprefix) - echo $(toolchain) - mkdir $(Build.ArtifactStagingDirectory)\app - copy C:\Windows\System32\calc.exe $(Build.ArtifactStagingDirectory)\app\example1.exe - copy C:\Windows\System32\calc.exe $(Build.ArtifactStagingDirectory)\app\example2.exe - copy C:\Windows\System32\calc.exe $(Build.ArtifactStagingDirectory)\app\example3.exe - displayName: 'Dummy build' - # - # To sign Windows binaries with ESRP, call esrpsign.sh - # with the files to sign as arguments. Requires the - # following environment variables to be set: - # ESRP_TOOL - set by the setup template above - # ESRP_AUTH - set by the setup template above - # SYSTEM_ACCESSTOKEN - $(System.AccessToken) - # - - ${{ if eq(parameters.esrp, true) }}: - bash: | + set -euo pipefail .azure-pipelines/esrp/windows/esrpsign.sh \ - "$BUILD_ARTIFACTSTAGINGDIRECTORY/app/example1.exe" \ - "$BUILD_ARTIFACTSTAGINGDIRECTORY/app/example2.exe" \ - "$BUILD_ARTIFACTSTAGINGDIRECTORY/app/example3.exe" - displayName: 'Example ESRP signing' + artifacts/Git-*.exe \ + artifacts/PortableGit-*.exe + displayName: 'ESRP-sign installer and portable Git' env: ESRP_TOOL: $(ESRP_TOOL) ESRP_AUTH: $(ESRP_AUTH) SYSTEM_ACCESSTOKEN: $(System.AccessToken) - # TODO: put final artifacts under $(Build.ArtifactStagingDirectory)/_final - - script: | - mkdir $(Build.ArtifactStagingDirectory)\_final - xcopy /s /y $(Build.ArtifactStagingDirectory)\app $(Build.ArtifactStagingDirectory)\_final - displayName: 'Dummy collect artifacts' + - task: Bash@3 + displayName: 'Stage installer artifacts for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + + # Compute SHA-256 over the (possibly signed) + # binaries; if ESRP signing ran, this picks up + # the post-sign bytes, which is what we want to + # publish in the release notes. + openssl dgst -sha256 \ + artifacts/Git-*.exe \ + artifacts/PortableGit-*.exe \ + | sed 's/.* //' >artifacts/sha-256.txt + + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + cp artifacts/Git-*.exe \ + artifacts/PortableGit-*.exe \ + artifacts/sha-256.txt \ + "$(Build.ArtifactStagingDirectory)/_final/" # # macOS build jobs @@ -239,20 +369,180 @@ extends: artifactName: '${{ dim.id }}' steps: - checkout: self - # TODO: add tasks to set up build environment - # TODO: add tasks to build Git and installers - - script: | - echo "Hello, Mac!" - mkdir -p $(Build.ArtifactStagingDirectory)/app - cp /bin/echo $(Build.ArtifactStagingDirectory)/app/example - displayName: 'Dummy build' + - task: Bash@3 + displayName: 'Install build dependencies' + inputs: + targetType: inline + script: | + set -euo pipefail + + # The agent's native arm64 Homebrew lives at + # /opt/homebrew. Install a separate x86_64 Homebrew + # under /usr/local so we can fetch the x86_64 build + # of gettext/libintl alongside the arm64 one. + arch -x86_64 /bin/bash -c \ + "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + arch -x86_64 /usr/local/bin/brew install gettext + + # Native (arm64) build dependencies. + brew install automake asciidoc xmlto docbook + brew link --force gettext + + # Make a universal libintl.a out of the two + # arch-specific copies, dropped at the workspace + # root so the build's LDFLAGS = -L"$(pwd)" can find + # it. libintl depends on iconv, but we deliberately + # use the system's universal /usr/lib/libiconv.dylib + # rather than Homebrew's libiconv (which exports + # _libiconv* symbols, while Homebrew's gettext was + # built against system iconv with _iconv* symbols). + lipo -create \ + -output libintl.a \ + /usr/local/opt/gettext/lib/libintl.a \ + /opt/homebrew/opt/gettext/lib/libintl.a + - task: Bash@3 + displayName: 'Configure universal build' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + # Git's GIT-VERSION-GEN expects .rc rather than -rc + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + echo "$BUILD_VERSION" >version + + # HOST_CPU is a bit of a lie and is only used in + # 'git version --build-options'; we'll fix that in + # code. The two -arch flags are what actually drive + # the universal build. + cat >config.mak <>config.mak <>config.mak <>config.mak + - task: Bash@3 + displayName: 'Build payload via macos-installer' + env: + # The macos-installer Makefile derives BUILD_DIR from + # $(GITHUB_WORKSPACE), which is unset in ADO. Point it + # at the worktree root. + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + + # The asciidoc/xmlto build steps need the catalogs + # from Homebrew docbook. + export XML_CATALOG_FILES="$(brew --prefix)/etc/xml/catalog" + + make -j"$(sysctl -n hw.physicalcpu)" GIT-VERSION-FILE dist dist-doc + + # Recover the source-tree commit OID from the dist + # tarball; the macos-installer Makefile bakes it + # into 'git version --build-options'. + GIT_BUILT_FROM_COMMIT="$(gunzip -c "git-$BUILD_VERSION.tar.gz" \ + | git get-tar-commit-id)" + export GIT_BUILT_FROM_COMMIT + export VERSION + + mkdir payload manpages + tar -xf "git-$BUILD_VERSION.tar.gz" -C payload + tar -xf "git-manpages-$BUILD_VERSION.tar.gz" -C manpages + + # The actual compile happens inside the extracted + # tree, against a copy of the config.mak we wrote + # at the worktree root in the previous step. + cp config.mak "payload/git-$BUILD_VERSION/config.mak" + + make -C .github/macos-installer V=1 payload + # ESRP-sign the universal Mach-O binaries inside the + # payload tree. The existing esrp/sign.yml template's + # CopyFiles@2 step uses minimatch globs and the only + # reliable way to detect Mach-O is by file content + # (file --mime), so we pre-filter into a staging dir, + # let the template zip/sign/extract that staging dir, + # then copy the signed binaries back over the payload. - ${{ if eq(parameters.esrp, true) }}: + # ESRP ADO tasks require .NET, which the macOS pool + # image does not provide by default. + - task: UseDotNet@2 + displayName: 'Install .NET for ESRP' + inputs: + packageType: sdk + version: '8.x' + - task: Bash@3 + displayName: 'Stage Mach-O binaries for signing' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + payload_dir="payload/git-$BUILD_VERSION" + stage_dir="$(Build.ArtifactStagingDirectory)/macos-tosign/binaries" + + rm -rf "$stage_dir" + mkdir -p "$stage_dir" + + pushd "$payload_dir" + find . -type f -exec file --mime {} + \ + | sed -n '/mach/s/: .*//p' \ + | while IFS= read -r f; do + rel="${f#./}" + tgt="$stage_dir/$rel" + mkdir -p "$(dirname "$tgt")" + cp -- "$f" "$tgt" + done + popd - template: .azure-pipelines/esrp/sign.yml@self parameters: - displayName: 'Example sign binaries' - folderPath: '$(Build.ArtifactStagingDirectory)/app' + displayName: 'ESRP-sign Mach-O binaries' + folderPath: '$(Build.ArtifactStagingDirectory)/macos-tosign/binaries' pattern: '**/*' - useArchive: true # Must be true when macOS signing + useArchive: true # Required for macOS signing inlineOperation: | [ { @@ -265,11 +555,101 @@ extends: } } ] - # TODO: put final artifacts under $(Build.ArtifactStagingDirectory)/_final - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/_final - cp -R $(Build.ArtifactStagingDirectory)/app/* $(Build.ArtifactStagingDirectory)/_final/ - displayName: 'Dummy collect artifacts' + - task: Bash@3 + displayName: 'Copy signed binaries back to payload' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + BUILD_VERSION="$(echo "$VERSION" | sed 's/-rc/.rc/g')" + cp -R "$(Build.ArtifactStagingDirectory)/macos-tosign/binaries"/* \ + "payload/git-$BUILD_VERSION/" + - task: Bash@3 + displayName: 'Build unsigned installer pkg' + env: + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + export VERSION + + # Leave APPLE_INSTALLER_IDENTITY undefined so the + # Makefile's `pkg` target produces an unsigned .pkg + # (the `ifdef APPLE_INSTALLER_IDENTITY` branch in + # pkg_cmd is skipped). ESRP signs it in the next + # step. + make -C .github/macos-installer V=1 pkg + - ${{ if eq(parameters.esrp, true) }}: + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-sign installer pkg' + folderPath: '.github/macos-installer/disk-image' + pattern: '*.pkg' + useArchive: true # Required for macOS signing + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppDeveloperSign", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "Hardening": "Enable" + } + } + ] + - template: .azure-pipelines/esrp/sign.yml@self + parameters: + displayName: 'ESRP-notarize installer pkg' + folderPath: '.github/macos-installer/disk-image' + pattern: '*.pkg' + useArchive: true # Required for macOS notarization + inlineOperation: | + [ + { + "KeyCode": "CP-401337-Apple", + "OperationCode": "MacAppNotarize", + "ToolName": "sign", + "ToolVersion": "1.0", + "Parameters": { + "BundleId": "com.git.pkg" + } + } + ] + - task: Bash@3 + displayName: 'Build DMG' + env: + GITHUB_WORKSPACE: $(Build.SourcesDirectory) + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + export VERSION + + # Builds .github/macos-installer/git--universal.dmg + # from the contents of disk-image/, which now contains + # the signed and notarized .pkg. + make -C .github/macos-installer V=1 image + - task: Bash@3 + displayName: 'Stage installer artifacts for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + + VERSION="$(git_version)" + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + mv ".github/macos-installer/git-$VERSION-universal.dmg" \ + "$(Build.ArtifactStagingDirectory)/_final/" + mv ".github/macos-installer/disk-image/git-$VERSION-universal.pkg" \ + "$(Build.ArtifactStagingDirectory)/_final/" # # Linux build jobs @@ -295,23 +675,72 @@ extends: artifactName: '${{ dim.id }}' steps: - checkout: self - # TODO: add tasks to set up build environment - # TODO: add tasks to build Git and installers - - script: | - echo $(cc_arch) - echo $(deb_arch) - mkdir -p $(Build.ArtifactStagingDirectory)/app - debroot=$(Build.ArtifactStagingDirectory)/pkgroot - mkdir -p $debroot/DEBIAN - cat > $debroot/DEBIAN/control <version + make GIT-VERSION-FILE + + PKGNAME="microsoft-git_${VERSION}_$(deb_arch)" + PKGDIR="$(Build.ArtifactStagingDirectory)/pkgroot" + rm -rf "$PKGDIR" + mkdir -p "$PKGDIR/DEBIAN" + + DESTDIR="$PKGDIR" make -j"$(nproc)" V=1 DEVELOPER=1 \ + USE_LIBPCRE=1 \ + USE_CURL_FOR_IMAP_SEND=1 NO_OPENSSL=1 \ + NO_CROSS_DIRECTORY_HARDLINKS=1 \ + ASCIIDOC8=1 ASCIIDOC_NO_ROFF=1 \ + ASCIIDOC='TZ=UTC asciidoc' \ + prefix=/usr/local \ + gitexecdir=/usr/local/lib/git-core \ + libexecdir=/usr/local/lib/git-core \ + htmldir=/usr/local/share/doc/git/html \ + install install-doc install-html + + # Based on https://packages.ubuntu.com/xenial/vcs/git + cat >"$PKGDIR/DEBIAN/control" < + Description: Git client built from the https://github.com/microsoft/git repository, + specialized in supporting monorepo scenarios. Includes the Scalar CLI. + CTRL + + mkdir -p "$(Build.ArtifactStagingDirectory)/app" + dpkg-deb -Zxz --build "$PKGDIR" \ + "$(Build.ArtifactStagingDirectory)/app/$PKGNAME.deb" - ${{ if eq(parameters.esrp, true) }}: # ESRP ADO tasks require .NET, so we install it here since the # Linux images do not have it by default. @@ -335,11 +764,15 @@ extends: "Parameters": {} } ] - # TODO: put final artifacts under $(Build.ArtifactStagingDirectory)/_final - - script: | - mkdir -p $(Build.ArtifactStagingDirectory)/_final - cp -R $(Build.ArtifactStagingDirectory)/app/* $(Build.ArtifactStagingDirectory)/_final/ - displayName: 'Dummy collect artifacts' + - task: Bash@3 + displayName: 'Stage Debian package for upload' + inputs: + targetType: inline + script: | + set -euo pipefail + mkdir -p "$(Build.ArtifactStagingDirectory)/_final" + mv "$(Build.ArtifactStagingDirectory)/app/microsoft-git_$(git_version)_$(deb_arch).deb" \ + "$(Build.ArtifactStagingDirectory)/_final/" - stage: release displayName: 'Release'