From b87096b5006790004659f95ce167c94c990f4a68 Mon Sep 17 00:00:00 2001 From: Dean Sheather Date: Wed, 8 Jun 2022 01:24:46 +1000 Subject: [PATCH] chore: split release workflow so the majority happens on Linux (#2092) --- .github/.goreleaser-release-darwin.yaml | 58 ++++++++++ .github/.goreleaser-release-linux.yaml | 55 +++++++++ .github/.goreleaser-release.yaml | 105 +++++++++++++++++ .github/workflows/coder.yaml | 6 +- .github/workflows/release.yaml | 145 +++++++++++++++++++++--- .goreleaser.yaml | 87 +------------- scripts/sign_macos.sh | 45 +++++--- 7 files changed, 381 insertions(+), 120 deletions(-) create mode 100644 .github/.goreleaser-release-darwin.yaml create mode 100644 .github/.goreleaser-release-linux.yaml create mode 100644 .github/.goreleaser-release.yaml diff --git a/.github/.goreleaser-release-darwin.yaml b/.github/.goreleaser-release-darwin.yaml new file mode 100644 index 0000000000000..1c6ea3f5d20d3 --- /dev/null +++ b/.github/.goreleaser-release-darwin.yaml @@ -0,0 +1,58 @@ +before: + hooks: + - go mod tidy + - rm -f site/out/bin/coder* + +archives: + - id: coder-darwin + builds: [coder-darwin] + format: zip + +builds: + - id: coder-slim + dir: cmd/coder + ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] + env: [CGO_ENABLED=0] + goos: [darwin, linux, windows] + goarch: [amd64, arm, arm64] + goarm: ["7"] + # Only build arm 7 for Linux + ignore: + - goos: windows + goarm: "7" + - goos: darwin + goarm: "7" + hooks: + # The "trimprefix" appends ".exe" on Windows. + post: | + cp {{.Path}} site/out/bin/coder-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ trimprefix .Name "coder" }} + + - id: coder-darwin + dir: cmd/coder + flags: [-tags=embed] + ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] + env: [CGO_ENABLED=0] + goos: [darwin] + goarch: [amd64, arm64] + hooks: + # This signs the binary that will be located inside the zip. MacOS + # requires the binary to be signed for notarization. + post: | + sh -c 'codesign -s {{.Env.AC_APPLICATION_IDENTITY}} -f -v --timestamp --options runtime {{.Path}}' + +env: + # Apple identity for signing! + - AC_APPLICATION_IDENTITY=BDB050EB749EDD6A80C6F119BF1382ECA119CCCC + +signs: + - ids: [coder-darwin] + artifacts: archive + cmd: ./scripts/sign_macos.sh + args: ["${artifact}"] + output: true + +release: + ids: [coder-darwin] + +snapshot: + name_template: "{{ .Version }}-devel+{{ .ShortCommit }}" diff --git a/.github/.goreleaser-release-linux.yaml b/.github/.goreleaser-release-linux.yaml new file mode 100644 index 0000000000000..d2d20a322ea6a --- /dev/null +++ b/.github/.goreleaser-release-linux.yaml @@ -0,0 +1,55 @@ +before: + hooks: + - go mod tidy + - rm -f site/out/bin/coder* + +archives: + - id: coder-linux + builds: [coder-linux] + format: tar.gz + + - id: coder-windows + builds: [coder-windows] + format: zip + +builds: + - id: coder-slim + dir: cmd/coder + ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] + env: [CGO_ENABLED=0] + goos: [darwin, linux, windows] + goarch: [amd64, arm, arm64] + goarm: ["7"] + # Only build arm 7 for Linux + ignore: + - goos: windows + goarm: "7" + - goos: darwin + goarm: "7" + hooks: + # The "trimprefix" appends ".exe" on Windows. + post: | + cp {{.Path}} site/out/bin/coder-{{ .Os }}-{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ trimprefix .Name "coder" }} + + - id: coder-linux + dir: cmd/coder + flags: [-tags=embed] + ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] + env: [CGO_ENABLED=0] + goos: [linux] + goarch: [amd64, arm, arm64] + goarm: ["7"] + + - id: coder-windows + dir: cmd/coder + flags: [-tags=embed] + ldflags: ["-s -w -X github.com/coder/coder/buildinfo.tag={{ .Version }}"] + env: [CGO_ENABLED=0] + goos: [windows] + goarch: [amd64, arm64] + +release: + ids: [coder-windows, coder-linux] + +snapshot: + name_template: "{{ .Version }}-devel+{{ .ShortCommit }}" diff --git a/.github/.goreleaser-release.yaml b/.github/.goreleaser-release.yaml new file mode 100644 index 0000000000000..ee050b830f9b1 --- /dev/null +++ b/.github/.goreleaser-release.yaml @@ -0,0 +1,105 @@ +# This goreleaser config file requires GoReleaser Pro as it uses the prebuilt +# builder type. + +archives: + - id: coder-linux + builds: [release-prebuilt-linux] + format: tar.gz + +builds: + - id: release-prebuilt-linux + builder: prebuilt + goos: [linux] + goarch: [amd64, arm, arm64] + goarm: ["7"] + prebuilt: + path: artifacts/coder-linux_{{.Os}}_{{.Arch}}{{ with .Arm }}_{{ . }}{{ end }}/coder + +# This section is also contained in .goreleaser.yaml. +nfpms: + - id: packages + vendor: Coder + homepage: https://coder.com + maintainer: Coder + description: | + Provision development environments with infrastructure with code + formats: + - apk + - deb + - rpm + suggests: + - postgresql + builds: + - release-prebuilt-linux + bindir: /usr/bin + contents: + - src: coder.env + dst: /etc/coder.d/coder.env + type: "config|noreplace" + - src: coder.service + dst: /usr/lib/systemd/system/coder.service + +# Image templates are empty on snapshots to avoid lengthy builds for +# development. +dockers: + - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-amd64{{ end }}"] + id: release-prebuilt-linux + dockerfile: Dockerfile + use: buildx + build_flag_templates: + - --platform=linux/amd64 + - --label=org.opencontainers.image.title=Coder + - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. + - --label=org.opencontainers.image.url=https://github.com/coder/coder + - --label=org.opencontainers.image.source=https://github.com/coder/coder + - --label=org.opencontainers.image.version={{ .Version }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=AGPL-3.0 + - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-arm64{{ end }}"] + goarch: arm64 + dockerfile: Dockerfile + use: buildx + build_flag_templates: + - --platform=linux/arm64/v8 + - --label=org.opencontainers.image.title=coder + - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. + - --label=org.opencontainers.image.url=https://github.com/coder/coder + - --label=org.opencontainers.image.source=https://github.com/coder/coder + - --label=org.opencontainers.image.version={{ .Tag }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=AGPL-3.0 + - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-armv7{{ end }}"] + goarch: arm + goarm: "7" + dockerfile: Dockerfile + use: buildx + build_flag_templates: + - --platform=linux/arm/v7 + - --label=org.opencontainers.image.title=Coder + - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. + - --label=org.opencontainers.image.url=https://github.com/coder/coder + - --label=org.opencontainers.image.source=https://github.com/coder/coder + - --label=org.opencontainers.image.version={{ .Tag }} + - --label=org.opencontainers.image.revision={{ .FullCommit }} + - --label=org.opencontainers.image.licenses=AGPL-3.0 + +docker_manifests: + - name_template: ghcr.io/coder/coder:{{ .Tag }} + image_templates: + - ghcr.io/coder/coder:{{ .Tag }}-amd64 + - ghcr.io/coder/coder:{{ .Tag }}-arm64 + - ghcr.io/coder/coder:{{ .Tag }}-armv7 + +release: + ids: [release-prebuilt-linux, packages] + footer: | + ## Container Image + - `docker pull ghcr.io/coder/coder:{{ .Tag }}` + # All non-Linux files should just be used as is. We have to import the Linux + # builds so that the docker images get built and package creation works. + extra_files: + - glob: ./artifacts/coder_*_darwin_*.zip + - glob: ./artifacts/coder_*_windows_*.zip + +snapshot: + name_template: "{{ .Version }}-devel+{{ .ShortCommit }}" diff --git a/.github/workflows/coder.yaml b/.github/workflows/coder.yaml index 39a20bba18348..8cbfac7756dcd 100644 --- a/.github/workflows/coder.yaml +++ b/.github/workflows/coder.yaml @@ -226,7 +226,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./gotests.coverage flags: unittest-go-${{ matrix.os }} - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false test-go-postgres: @@ -310,7 +310,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./gotests.coverage flags: unittest-go-postgres-${{ matrix.os }} - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false deploy: @@ -437,7 +437,7 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: ./site/coverage/lcov.info flags: unittest-js - # this flakes and sometimes fails the build + # this flakes and sometimes fails the build fail_ci_if_error: false - name: Upload DataDog Trace diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1736a45ce1d9d..0a7025130b24e 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -1,33 +1,87 @@ +# GitHub release workflow. +# +# This workflow is a bit complicated because we have to build darwin binaries on +# a mac runner, but the mac runners are extremely slow. So instead of running +# the entire release on a mac (which will take an hour to run), we run only the +# mac build on a mac, and the rest on a linux runner. The final release is then +# published using a final linux runner. name: release on: push: tags: - "v*" workflow_dispatch: + inputs: + snapshot: + description: Perform a snapshot/dry-run release (will not create a GitHub release, required if the ref is not a tag) + type: boolean + required: true jobs: - goreleaser: - runs-on: macos-latest - env: - # Necessary for Docker manifest - DOCKER_CLI_EXPERIMENTAL: "enabled" + linux-windows: + runs-on: ubuntu-latest steps: - # Docker is not included on macos-latest - - uses: docker-practice/actions-setup-docker@1.0.10 - - uses: actions/checkout@v3 with: fetch-depth: 0 - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + - uses: actions/setup-go@v3 + with: + go-version: "~1.18" - - name: Docker Login - uses: docker/login-action@v2 + - name: Echo Go Cache Paths + id: go-cache-paths + run: | + echo "::set-output name=go-build::$(go env GOCACHE)" + echo "::set-output name=go-mod::$(go env GOMODCACHE)" + + - name: Go Build Cache + uses: actions/cache@v3 with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} + path: ${{ steps.go-cache-paths.outputs.go-build }} + key: ${{ runner.os }}-release-go-build-${{ hashFiles('**/go.sum') }} + + - name: Go Mod Cache + uses: actions/cache@v3 + with: + path: ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-release-go-mod-${{ hashFiles('**/go.sum') }} + + - name: Cache Node + id: cache-node + uses: actions/cache@v3 + with: + path: | + **/node_modules + .eslintcache + key: js-${{ runner.os }}-test-${{ hashFiles('**/yarn.lock') }} + restore-keys: | + js-${{ runner.os }}- + + - name: Build Site + run: make site/out/index.html + + - name: Build Linux and Windows binaries with GoReleaser + uses: goreleaser/goreleaser-action@v3 + with: + version: latest + args: release -f ./.github/.goreleaser-release-linux.yaml --rm-dist --timeout 60m --skip-publish --skip-announce ${{ github.event.inputs.snapshot && '--snapshot' }} + + - name: Upload binary artifacts + uses: actions/upload-artifact@v3 + with: + name: linux + path: ./dist/coder* + + # The mac binaries get built on mac runners because they need to be signed, + # and the signing tool only runs on mac. This darwin job only builds the Mac + # binaries and uploads them as job artifacts used by the publish step. + darwin: + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 - uses: actions/setup-go@v3 with: @@ -79,12 +133,69 @@ jobs: - name: Build Site run: make site/out/index.html - - name: Run GoReleaser + - name: Build Darwin binaries with GoReleaser + uses: goreleaser/goreleaser-action@v3 + with: + version: latest + args: release -f ./.github/.goreleaser-release-darwin.yaml --rm-dist --timeout 60m --skip-publish --skip-announce ${{ github.event.inputs.snapshot && '--snapshot' }} + env: + AC_USERNAME: ${{ secrets.AC_USERNAME }} + AC_PASSWORD: ${{ secrets.AC_PASSWORD }} + + - name: Upload binary artifacts + uses: actions/upload-artifact@v3 + with: + name: darwin + path: ./dist/coder* + + publish: + runs-on: ubuntu-latest + needs: + - linux-windows + - darwin + env: + # Necessary for Docker manifest + DOCKER_CLI_EXPERIMENTAL: "enabled" + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Docker Login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: mkdir artifacts + run: mkdir artifacts + + - name: Download darwin artifacts + uses: actions/download-artifact@v3 + with: + name: darwin + path: ./artifacts + + - name: Download Linux and Windows artifacts + uses: actions/download-artifact@v3 + with: + name: linux + path: ./artifacts + + - name: ls ./artifacts + run: ls ./artifacts + + # This will build Docker images and Linux packages. + - name: Publish release uses: goreleaser/goreleaser-action@v3 with: + # we use the "prebuilt" builder here which is a pro-only feature + distribution: goreleaser-pro version: latest - args: release --rm-dist --timeout 60m + args: release -f ./.github/.goreleaser-release.yaml --rm-dist --timeout 60m ${{ github.event.inputs.snapshot && '--snapshot' }} env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AC_USERNAME: ${{ secrets.AC_USERNAME }} AC_PASSWORD: ${{ secrets.AC_PASSWORD }} diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7bcedf5087db5..602023d91e3d1 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -1,16 +1,3 @@ -archives: - - id: coder-linux - builds: [coder-linux] - format: tar.gz - - - id: coder-darwin - builds: [coder-darwin] - format: zip - - - id: coder-windows - builds: [coder-windows] - format: zip - before: hooks: - go mod tidy @@ -59,18 +46,8 @@ builds: env: [CGO_ENABLED=0] goos: [darwin] goarch: [amd64, arm64] - hooks: - # This signs the binary that will be located inside the zip. - # MacOS requires the binary to be signed for notarization. - # - # If it doesn't successfully sign, the zip sign step will error. - post: | - sh -c 'codesign -s {{.Env.AC_APPLICATION_IDENTITY}} -f -v --timestamp --options runtime {{.Path}} || true' - -env: - # Apple identity for signing! - - AC_APPLICATION_IDENTITY=BDB050EB749EDD6A80C6F119BF1382ECA119CCCC +# This section is also contained in .goreleaser-release.yaml. nfpms: - id: packages vendor: Coder @@ -94,67 +71,5 @@ nfpms: - src: coder.service dst: /usr/lib/systemd/system/coder.service -# Image templates are empty on snapshots to avoid lengthy builds for development. -dockers: - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-amd64{{ end }}"] - id: coder-linux - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/amd64 - - --label=org.opencontainers.image.title=Coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Version }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-arm64{{ end }}"] - goarch: arm64 - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/arm64/v8 - - --label=org.opencontainers.image.title=coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Tag }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 - - image_templates: ["{{ if not .IsSnapshot }}ghcr.io/coder/coder:{{ .Tag }}-armv7{{ end }}"] - goarch: arm - goarm: "7" - dockerfile: Dockerfile - use: buildx - build_flag_templates: - - --platform=linux/arm/v7 - - --label=org.opencontainers.image.title=Coder - - --label=org.opencontainers.image.description=A tool for provisioning self-hosted development environments with Terraform. - - --label=org.opencontainers.image.url=https://github.com/coder/coder - - --label=org.opencontainers.image.source=https://github.com/coder/coder - - --label=org.opencontainers.image.version={{ .Tag }} - - --label=org.opencontainers.image.revision={{ .FullCommit }} - - --label=org.opencontainers.image.licenses=AGPL-3.0 -docker_manifests: - - name_template: ghcr.io/coder/coder:{{ .Tag }} - image_templates: - - ghcr.io/coder/coder:{{ .Tag }}-amd64 - - ghcr.io/coder/coder:{{ .Tag }}-arm64 - - ghcr.io/coder/coder:{{ .Tag }}-armv7 - -release: - ids: [coder-linux, coder-darwin, coder-windows, packages] - footer: | - ## Container Image - - `docker pull ghcr.io/coder/coder:{{ .Tag }}` - -signs: - - ids: [coder-darwin] - artifacts: archive - cmd: ./scripts/sign_macos.sh - args: ["${artifact}"] - output: true - snapshot: name_template: "{{ .Version }}-devel+{{ .ShortCommit }}" diff --git a/scripts/sign_macos.sh b/scripts/sign_macos.sh index 26d3fd181bbcb..523649d7c5cf4 100755 --- a/scripts/sign_macos.sh +++ b/scripts/sign_macos.sh @@ -5,19 +5,36 @@ set -euo pipefail SCRIPT_DIR=$(dirname "${BASH_SOURCE[0]}") PROJECT_ROOT=$(cd "$SCRIPT_DIR" && git rev-parse --show-toplevel) -( - cd "${PROJECT_ROOT}" +cd "${PROJECT_ROOT}" - codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$1" +codesign -s "$AC_APPLICATION_IDENTITY" -f -v --timestamp --options runtime "$1" - config=$(mktemp -d)/gon.json - jq -r --null-input --arg path "$(pwd)/$1" '{ - "notarize": [ - { - "path": $path, - "bundle_id": "com.coder.cli" - } - ] - }' >"$config" - gon "$config" -) +config=$(mktemp -d)/gon.json +jq -r --null-input --arg path "$(pwd)/$1" '{ + "notarize": [ + { + "path": $path, + "bundle_id": "com.coder.cli" + } + ] +}' >"$config" + +# The notarization process is very fragile and heavily dependent on Apple's +# notarization server not returning server errors, so we retry this step 5 +# times with a delay of 30 seconds between each attempt. +rc=0 +for i in $(seq 1 5); do + gon "$config" && rc=0 && break || rc=$? + echo "gon exit code: $rc" + if [ "$i" -lt 5 ]; then + echo + echo "Retrying notarization in 30 seconds" + echo + sleep 30 + else + echo + echo "Giving up :(" + fi +done + +exit $rc