master-build.yml

# Signs windows and macOS binaries and installers
name: Master branch build/release (signed)
defaults:
  run:
    shell: bash
on:
  push:
    branches: [ master, main ]
  release:
    types: [ created ]
  workflow_dispatch:
    inputs:
      debug_enabled:
        description: 'Run the build with tmate set "debug_enabled"'
        type: boolean
        required: false
        default: false

# We do not cancel in-progress builds here because in general we
# want to see success on each commit, and we usually use [skip ci]
# to prevent builds on commits that don't matter.
#concurrency:
#  group: ${{ github.workflow }}-${{ github.ref }}
#  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

env:
  BUILDKIT_PROGRESS: plain
  DOCKER_CLI_EXPERIMENTAL: enabled
  DDEV_DEBUG: true
  GITHUB_REPOSITORY_OWNER: ${{ secrets.GITHUB_REPOSITORY_OWNER }}
  DDEV_WINDOWS_SIGN: ${{ secrets.DDEV_WINDOWS_SIGN }}
  HOMEBREW_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

jobs:
  build-most:
    name: Build DDEV executables except Windows
    runs-on: ubuntu-22.04
    env:
      AmplitudeAPIKey: ${{ secrets.AMPLITUDE_API_KEY }}
    steps:
      - uses: actions/checkout@v4
        with:
          # We need to get all branches and tags for git describe to work properly
          fetch-depth: 0

      - name: Set up Homebrew
        id: set-up-homebrew
        uses: Homebrew/actions/setup-homebrew@master

      - name: Install Docker and deps
        run: ./.github/workflows/linux-setup.sh

      - name: Install build tools
        run: ./.github/workflows/linux-build-setup.sh

      - uses: actions/setup-go@v5
        with:
          go-version: '>=1.21'

      - name: Build DDEV executables
        run: make linux_amd64 linux_arm64 darwin_amd64 darwin_arm64 completions mkcert

      - name: "Verify that instrumentation is working (Linux amd64)"
        run: |
          export PATH=".gotmp/bin/linux_amd64:$PATH"
          echo "DDEV_NO_INSTRUMENTATION=${DDEV_NO_INSTRUMENTATION}"
          if [ -z "${AmplitudeAPIKey}" ]; then echo "AmplitudeAPIKey is not set"; exit 1; fi
          ddev config global --instrumentation-opt-in=true
          ddev config global | grep instrumentation-opt-in=true
          ddev version | grep -v "AmplitudeAPIKey is not available."
          ddev config global --instrumentation-opt-in=false


      - name: save build results to cache
        uses: actions/cache@v4
        with:
          path: .gotmp/bin
          key: ${{ github.sha }}-${{ github.ref }}-build-most

  # This Windows self-hosted runner has to be set up with gnu tar and zstd.exe, or
  # this step will fail to properly create the cache.
  # Make sure gnu tar is the tar used here. System PATH should have C:\program files\gnu\usr\bin near top
  # Get zstd.exe from https://github.com/facebook/zstd/releases - I put it into C:\program files\gnu\usr\bin
  # so it would be in PATH
  # See https://github.com/actions/cache/issues/580
  # Run the Windows action with debug enabled to be able to see which tar is in use, etc.
  sign-windows:
    name: Build and Sign Windows binaries
    runs-on: [ self-hosted, windows-signer ]
    env:
      AmplitudeAPIKey: ${{ secrets.AMPLITUDE_API_KEY }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Build and sign windows amd64 + arm64 binaries and amd64 installer
        shell: bash
        run: |
          if [ "${DDEV_WINDOWS_SIGN}" != "true" ]; then echo "Warning: DDEV_WINDOWS_SIGN is not true"; fi
          make windows_install
      - name: Show github.ref
        run: echo ${{ github.ref }}
      - name: Build chocolatey on release
        if: startsWith( github.ref, 'refs/tags/v1')
        run: make chocolatey

      - name: "Verify that intrumentation is working (Windows)"
        run: |
          export PATH=".gotmp/bin/windows_amd64:$PATH"
          echo "DDEV_NO_INSTRUMENTATION=${DDEV_NO_INSTRUMENTATION}"
          if [ -z "${AmplitudeAPIKey}" ]; then echo "AmplitudeAPIKey is not set"; exit 1; fi
          ddev config global --instrumentation-opt-in=true
          ddev config global | grep instrumentation-opt-in=true
          ddev version | grep -v "AmplitudeAPIKey is not available."
          ddev config global --instrumentation-opt-in=false

      - name: Cache signed binaries
        # After 3.0.5 they were using zstdmt which is not available on Windows
        # See https://github.com/actions/cache/issues/891
        # But trying again with @v3, 2023-02-18
        uses: actions/cache@v4
        with:
          path: .gotmp/bin/windows*
          key: ${{ github.sha }}-${{ github.ref }}-signed-windows-binaries
          enableCrossOsArchive: true

  notarize-macos:
    name: Sign and Notarize ddev on macOS
    runs-on: macos-latest
    needs: build-most
    steps:
      - name: "setup macOS"
        run: |
          brew install coreutils gnu-getopt jq
      - uses: actions/checkout@v4
        with:
          # We need to get all branches and tags for git describe to work properly
          fetch-depth: 0
      - uses: actions/setup-go@v5
        with:
          go-version: '>=1.21'

      - name: restore build-most results from cache
        uses: actions/cache@v4
        id: buildmost
        with:
          path: .gotmp/bin
          key: ${{ github.sha }}-${{ github.ref }}-build-most
          fail-on-cache-miss: true
      - name: test that buildmost cache was loaded
        if: steps.buildmost.outputs.cache-hit != 'true'
        run: exit 1

      - name: Sign and notarize binaries (amd64 and arm64)
        env:
          DDEV_MACOS_SIGNING_PASSWORD: ${{ secrets.DDEV_MACOS_SIGNING_PASSWORD }}
          DDEV_MACOS_APP_PASSWORD: ${{ secrets.DDEV_MACOS_APP_PASSWORD }}
          TEAM_ID: "9HQ298V2BW"
        run: |
          set -o errexit -o pipefail
          if [ -z "${DDEV_MACOS_SIGNING_PASSWORD}" ] ; then echo "DDEV_MACOS_SIGNING_PASSWORD is empty"; exit 1; fi
          if [ -z "${DDEV_MACOS_APP_PASSWORD}" ] ; then echo "DDEV_MACOS_APP_PASSWORD is empty"; exit 1; fi
          for item in .gotmp/bin/darwin*/ddev; do
            echo "Signing and notarizing ${item} ..."
            codesign --remove-signature "${item}" || true
            curl -s https://raw.githubusercontent.com/ddev/signing_tools/master/macos_sign.sh | bash -s -  --signing-password="${DDEV_MACOS_SIGNING_PASSWORD}" --cert-file=certfiles/ddev_developer_id_cert.p12 --cert-name="Developer ID Application: Localdev Foundation (9HQ298V2BW)" --target-binary="${item}"
            curl -sSL -f https://raw.githubusercontent.com/ddev/signing_tools/master/macos_notarize.sh | bash -s -  --app-specific-password=${DDEV_MACOS_APP_PASSWORD} --apple-id=notarizer@localdev.foundation --primary-bundle-id=com.ddev.ddev --target-binary="${item}"
          done
      - name: Save notarized binaries to cache
        uses: actions/cache@v4
        with:
          path: .gotmp/bin/darwin*
          key: ${{ github.sha }}-${{ github.ref }}-notarize-macos

  artifacts:
    name: Upload artifacts
    runs-on: ubuntu-22.04
    needs: [build-most, sign-windows, notarize-macos]
    env:
      DDEV_GITHUB_TOKEN: ${{ secrets.DDEV_GITHUB_TOKEN }}
      CHOCOLATEY_API_KEY: ${{ secrets.CHOCOLATEY_API_KEY }}

    steps:
      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        with:
          limit-access-to-actor: true
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}

      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Restore build-most builds
        uses: actions/cache@v4
        id: buildmost
        with:
          path: .gotmp/bin
          key: ${{ github.sha }}-${{ github.ref }}-build-most
          enableCrossOsArchive: true
          fail-on-cache-miss: true
      - name: test that build-most was loaded
        if: steps.buildmost.outputs.cache-hit != 'true'
        run: exit 1

      - name: Restore Signed Windows artifacts
        uses: actions/cache@v4
        id: signedwindows
        with:
          path: .gotmp/bin/windows*
          key: ${{ github.sha }}-${{ github.ref }}-signed-windows-binaries
          enableCrossOsArchive: true
          fail-on-cache-miss: true
      - name: test that signed-windows was loaded
        if: steps.signedwindows.outputs.cache-hit != 'true'
        run: exit 1

      - name: Restore Signed Mac artifacts
        uses: actions/cache@v4
        id: notarizedmac
        with:
          path: .gotmp/bin/darwin*
          key: ${{ github.sha }}-${{ github.ref }}-notarize-macos
          enableCrossOsArchive: true
          fail-on-cache-miss: true
      - name: test that notarizedmac was loaded
        if: steps.notarizedmac.outputs.cache-hit != 'true'
        run: exit 1

      - name: "Verify that cgo_enabled is 0 in Linux binary"
        run: |
          .gotmp/bin/$(go env GOOS)_$(go env GOARCH)/ddev version
          cgo=$(".gotmp/bin/$(go env GOOS)_$(go env GOARCH)/ddev" version 2>/dev/null | awk '/cgo_enabled/ {print $2}')
          if [ "${cgo}" != "0" ]; then echo "CGO_ENABLED=${cgo} but it must be 0 in released binary" && exit 10; fi

      # Goreleaser does GitHub release artifacts, homebrew, AUR, deb/rpm
      - name: goreleaser
        uses: goreleaser/goreleaser-action@v6
        if: startsWith( github.ref, 'refs/tags/v1')
        with:
          distribution: goreleaser-pro
          version: latest
          args: release --clean
        env:
          CGO_ENABLED: 0
          DOCKER_ORG: ddev
          GITHUB_TOKEN: ${{ secrets.DDEV_GITHUB_TOKEN }}
          GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }}
          FURY_ACCOUNT: ${{ secrets.FURY_ACCOUNT }}
          FURY_TOKEN: ${{ secrets.FURY_TOKEN }}
          AUR_SSH_PRIVATE_KEY: ${{ secrets.AUR_SSH_PRIVATE_KEY }}
          AUR_STABLE_GIT_URL: ${{ secrets.AUR_STABLE_GIT_URL }}
          AUR_EDGE_GIT_URL: ${{ secrets.AUR_EDGE_GIT_URL }}

      # Do artifacts for upload to workflow URL
      - name: Generate artifacts
        run: ./.ci-scripts/generate_artifacts.sh ${{ github.workspace }}/artifacts

      - name: Upload all artifacts
        uses: actions/upload-artifact@v4
        with:
          name: all-ddev-executables
          path: ${{ github.workspace }}/artifacts/*
      - name: Upload macos-amd64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-macos-amd64
          path: .gotmp/bin/darwin_amd64/ddev
      - name: Upload macos-arm64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-macos-arm64
          path: .gotmp/bin/darwin_arm64/ddev
      - name: Upload linux-arm64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-linux-arm64
          path: .gotmp/bin/linux_arm64/ddev
      - name: Upload linux-amd64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-linux-amd64
          path: .gotmp/bin/linux_amd64/ddev
      - name: Upload windows_amd64 installer
        uses: actions/upload-artifact@v4
        with:
          name: ddev-windows-amd64-installer
          path: .gotmp/bin/windows_amd64/ddev_windows_installer*.exe
      - name: Upload windows-arm64 binary
        uses: actions/upload-artifact@v4
        with:
          name: ddev-windows-arm64
          path: .gotmp/bin/windows_arm64/ddev.exe

      - name: Show github.ref
        run: echo ${{ github.ref }}

      - name: Chocolatey windows release
        if: env.CHOCOLATEY_API_KEY != '' && startsWith( github.ref, 'refs/tags/v1')
        run: |
          pushd .gotmp/bin/windows_amd64/chocolatey
          docker run --rm -v $PWD:/tmp/chocolatey -w /tmp/chocolatey linuturk/mono-choco push -s https://push.chocolatey.org/ --api-key "${CHOCOLATEY_API_KEY}"
          popd