From b3b2332e65199972a06d81a61d1b60f580644a21 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Thu, 22 Sep 2022 14:09:55 -0700 Subject: [PATCH 1/5] Use two stage release process, plus diamond workflow for cross-compiling --- .github/workflows/release.yml | 141 +++++++++++++++++++++++++++------- .github/workflows/stage.yml | 42 ++++++++++ cli/.gitignore | 3 +- cli/.goreleaser.yaml | 93 ---------------------- cli/Makefile | 47 +++++++++--- cli/combined-release.yml | 75 ++++++++++++++++++ cli/cross-release.yml | 53 +++++++++++++ cli/darwin-release.yml | 28 +++++++ 8 files changed, 350 insertions(+), 132 deletions(-) create mode 100644 .github/workflows/stage.yml delete mode 100644 cli/.goreleaser.yaml create mode 100644 cli/combined-release.yml create mode 100644 cli/cross-release.yml create mode 100644 cli/darwin-release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9574c6e57317b..f029b3ce8d81f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: Release +name: Release From Branch env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} @@ -6,45 +6,106 @@ env: on: workflow_dispatch: inputs: - increment: - description: "SemVer Increment" - required: true - default: "prerelease" - type: choice - options: - - prerelease - - prepatch - - preminor - - premajor - - patch - - minor - - major + release_branch: + description: "Staging branch to run release from" + jobs: - build: - runs-on: macos-latest + smoke-test: + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: - token: ${{ secrets.TURBOBOT }} + ref: ${{ inputs.release_branch }} - uses: ./.github/actions/setup-node with: enable-corepack: false - uses: ./.github/actions/setup-go with: github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Test + run: pnpm -- turbo run test --filter=cli --color - - name: golangci-lint - uses: golangci/golangci-lint-action@v3 + darwin: + needs: [smoke-test] + runs-on: macos-latest + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.release_branch }} + - run: git fetch origin --tags + - uses: ./.github/actions/setup-node with: - # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version + enable-corepack: false + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Install GoReleaser + uses: goreleaser/goreleaser-action@v3 + with: + distribution: goreleaser-pro version: latest + install-only: true + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + - name: Build Artifacts + run: cd cli && make publish-turbo-darwin + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: turbo-artifacts-darwin + path: cli/dist-darwin - # Optional: working directory, useful for monorepos - working-directory: cli + # compiles linux and windows in a container + cross: + needs: [smoke-test] + runs-on: ubuntu-latest + container: + image: docker://ghcr.io/gsoltis/turbo-cross:v1.18.5 + steps: + - uses: actions/checkout@v3 + with: + ref: "${{ inputs.release_branch }}" + - run: git fetch origin --tags + - uses: ./.github/actions/setup-node + with: + enable-corepack: false + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + - name: Install GoReleaser + uses: goreleaser/goreleaser-action@v3 + with: + distribution: goreleaser-pro + version: latest + install-only: true + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + - name: Build Artifacts + run: cd cli && make publish-turbo-cross + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + - name: Upload Artifacts + uses: actions/upload-artifact@v3 + with: + name: turbo-artifacts-cross + path: cli/dist-cross - # `golangci-lint-action` does not have an "install only" option. - # We ignore the output of this run, instead using it just to install the binary. - args: --issues-exit-code=0 + final-publish: + needs: [cross, darwin] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: "${{ inputs.release_branch }}" + - run: git fetch origin --tags + - uses: ./.github/actions/setup-node + with: + enable-corepack: false + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Configure git run: | @@ -60,12 +121,34 @@ jobs: env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} - - name: Version - run: | - ./scripts/version.js ${{ inputs.increment }} - cat version.txt + - name: Download Cross-compiled Artifacts + uses: actions/download-artifact@v3 + with: + name: turbo-artifacts-cross + path: cli/dist-cross + + - name: Download Darwin Artifacts + uses: actions/download-artifact@v3 + with: + name: turbo-artifacts-darwin + path: cli/dist-darwin + + - name: Combine Artifacts + run: cd cli && mkdir -p dist-combined && cp -a dist-cross/. dist-combined/ && cp -a dist-darwin/. dist-combined/ + env: + GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} - name: Release run: cd cli && make publish env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + + - uses: repo-sync/pull-request@v2 + with: + source_branch: ${{ inputs.release_branch }} + destination_branch: main + pr_title: Merge release branch ${{ inputs.release_branch }} + pr_body: ":crown: Merge release branch back to main" + pr_reviewer: ${{ github.actor }},gsoltis + pr_allow_empty: true + github_token: ${{ secrets.TURBOBOT }} diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml new file mode 100644 index 0000000000000..1c03ccd6c6294 --- /dev/null +++ b/.github/workflows/stage.yml @@ -0,0 +1,42 @@ +name: Create Release Branch + +# TODO: set outputs? optionally trigger release immediately? +on: + workflow_dispatch: + inputs: + increment: + description: "SemVer Increment" + required: true + default: "prerelease" + type: choice + options: + - prerelease + - prepatch + - preminor + - premajor + - patch + - minor + - major + +jobs: + stage: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + token: ${{ secrets.TURBOBOT }} + - uses: ./.github/actions/setup-node + with: + enable-corepack: false + - name: Configure git + run: | + git config --global user.name 'Turbobot' + git config --global user.email 'turbobot@vercel.com' + - name: Version + run: | + ./scripts/version.js ${{ inputs.increment }} + cat version.txt + - name: Stage Commit + run: cd cli && make stage-release && echo "STAGE_BRANCH=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Show Stage Commit + run: echo "${{ env.STAGE_BRANCH }}" diff --git a/cli/.gitignore b/cli/.gitignore index f2929414b9bc2..0175fb3d52d66 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -1,7 +1,8 @@ -/internal/turbodprotocol/*.go +#/internal/turbodprotocol/*.go /demo/ /dist/ +/dist-* # Built binaries. /turbo diff --git a/cli/.goreleaser.yaml b/cli/.goreleaser.yaml deleted file mode 100644 index 9dc100b3f2d66..0000000000000 --- a/cli/.goreleaser.yaml +++ /dev/null @@ -1,93 +0,0 @@ -project_name: turbo -before: - hooks: - - make compile-protos - - go mod tidy -builds: - - id: turbo - main: ./cmd/turbo - binary: bin/turbo - hooks: - pre: - - cmd: ./scripts/npm-native-packages/npm-native-packages.js {{ .Os }} {{ .Arch }} {{ .Version }} - flags: - - -trimpath - ldflags: - - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser - mod_timestamp: '{{ .CommitTimestamp }}' - env: - - CGO_ENABLED=0 - targets: - - linux_amd64 - - linux_arm64 - - windows_amd64 - - windows_arm64 - - id: turbo-cgo - main: ./cmd/turbo - binary: bin/turbo - hooks: - pre: - - cmd: ./scripts/npm-native-packages/npm-native-packages.js {{ .Os }} {{ .Arch }} {{ .Version }} - flags: - - -trimpath - ldflags: - - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser - mod_timestamp: '{{ .CommitTimestamp }}' - env: - - CGO_ENABLED=1 - targets: - - darwin_amd64 - - darwin_arm64 -checksum: - name_template: 'checksums.txt' -snapshot: - name_template: "{{ incpatch .Version }}" -archives: - - id: github - name_template: '{{ .ProjectName }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}' - wrap_in_directory: true - replacements: - amd64: 64 - format: tar.gz - format_overrides: - - goos: windows - format: zip - files: - - LICENSE - - README.md - - id: npm - name_template: '{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}' - wrap_in_directory: true - replacements: - amd64: 64 - format: tar.gz - files: - - LICENSE - - src: 'scripts/npm-native-packages/build/{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}/package.json' - dst: 'workaround/..' - strip_parent: true - - src: 'scripts/npm-native-packages/build/{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}/README.md' - dst: 'workaround/..' - strip_parent: true - - src: 'scripts/npm-native-packages/build/{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}/bin/*' - dst: 'bin/' - strip_parent: true -changelog: - sort: asc - filters: - exclude: - - '^docs:' - - '^test:' -release: - github: - owner: vercel - name: turborepo - ids: - - github - prerelease: auto - disable: true -publishers: - - name: npm - ids: - - npm - cmd: 'npm publish{{ if .Prerelease }} --tag canary{{ end }} {{ abs .ArtifactPath }}' diff --git a/cli/Makefile b/cli/Makefile index 912e8dc93ab1d..fecf15e944ee3 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -78,17 +78,42 @@ cmd/turbo/version.go: ../version.txt node -e 'console.log(`package main\n\nconst turboVersion = "$(TURBO_VERSION)"`)' > cmd/turbo/version.go.txt mv cmd/turbo/version.go.txt cmd/turbo/version.go -build: install compile-protos cmd/turbo/version.go $(GENERATED_FILES) +build: install # cd $(CLI_DIR)/../packages/turbo-ignore && pnpm install --filter=turbo-ignore && npm run build cd $(CLI_DIR)/../packages/create-turbo && pnpm install --filter=create-turbo && npm run build cd $(CLI_DIR)/../packages/turbo-codemod && pnpm install --filter=@turbo/codemod && npm run build -prepublish: - make -j3 bench/turbo test-go lint-go +.PHONY: prepublish +prepublish: compile-protos cmd/turbo/version.go + make -j3 bench/turbo test-go -publish: clean prepublish build +.PHONY: publish-turbo-cross +publish-turbo-cross: prepublish + goreleaser release --rm-dist -f cross-release.yml + +.PHONY: publish-turbo-darwin +publish-turbo-darwin: prepublish + goreleaser release --rm-dist -f darwin-release.yml + +.PHONY: snapshot-turbo-cross +snapshot-turbo-cross: + goreleaser release --snapshot --rm-dist -f cross-release.yml + +.PHONY: snapshot-turbo-darwin +snapshot-turbo-darwin: + goreleaser release --snapshot --rm-dist -f darwin-release.yml + +.PHONY: snapshot-turbo +snapshot-turbo: clean + goreleaser release --snapshot --rm-dist -f combined-release.yml + +.PHONY: stage-release +stage-release: cmd/turbo/version.go echo "Version: $(TURBO_VERSION)" echo "Tag: $(TURBO_TAG)" + cat $(CLI_DIR)/../version.txt + git diff -- $(CLI_DIR)/../version.txt + git status @test main = "`git rev-parse --abbrev-ref HEAD`" || (echo "Refusing to publish from non-main branch `git rev-parse --abbrev-ref HEAD`" && false) @test "" = "`git cherry`" || (echo "Refusing to publish with unpushed commits" && false) @@ -102,16 +127,24 @@ publish: clean prepublish build cd $(CLI_DIR)/../packages/create-turbo && npm version "$(TURBO_VERSION)" --allow-same-version cd $(CLI_DIR)/../packages/turbo-codemod && npm version "$(TURBO_VERSION)" --allow-same-version + git checkout -b staging-$(TURBO_VERSION) git commit -anm "publish $(TURBO_VERSION) to registry" git tag "v$(TURBO_VERSION)" + git push origin staging-$(TURBO_VERSION) --tags --force + +.PHONY: publish +publish: clean build + echo "Version: $(TURBO_VERSION)" + echo "Tag: $(TURBO_TAG)" # Include the patch in the log. git format-patch HEAD~1 --stdout | cat npm config set --location=project "//registry.npmjs.org/:_authToken" $(NPM_TOKEN) + # Publishes the native npm modules. - goreleaser release --rm-dist + goreleaser release --rm-dist -f combined-release.yml # Split packing from the publish step so that npm locates the correct .npmrc file. npm pack $(CLI_DIR)/../packages/turbo --pack-destination=$(CLI_DIR)/../ @@ -126,10 +159,6 @@ publish: clean prepublish build npm publish -ddd --tag $(TURBO_TAG) $(CLI_DIR)/../create-turbo-$(TURBO_VERSION).tgz npm publish -ddd --tag $(TURBO_TAG) $(CLI_DIR)/../turbo-codemod-$(TURBO_VERSION).tgz - # Force push to ensure that the version used for the release is published. - # This _intentionally_ clobbers. If it does so, redo the clobbered thing. - git push -f origin main "v$(TURBO_VERSION)" - demo/lage: install node $(CLI_DIR)/scripts/generate.mjs lage diff --git a/cli/combined-release.yml b/cli/combined-release.yml new file mode 100644 index 0000000000000..a63c1764993c6 --- /dev/null +++ b/cli/combined-release.yml @@ -0,0 +1,75 @@ +project_name: turbo + +dist: dist + +builds: + - id: turbo + builder: prebuilt + goos: + - linux + - windows + - darwin + goarch: + - amd64 + - arm64 + goamd64: + - v1 + prebuilt: + path: dist-combined/turbo_{{ .Os }}_{{ .Arch }}{{ with .Amd64 }}_{{ .}}{{ end }}/bin/turbo{{ .Ext }} + hooks: + pre: + - cmd: ./scripts/npm-native-packages/npm-native-packages.js {{ .Os }} {{ .Arch }} {{ .Version }} + binary: turbo +checksum: + name_template: "checksums.txt" +snapshot: + name_template: "{{ incpatch .Version }}" +archives: + - id: github + name_template: "{{ .ProjectName }}-{{ .Version }}-{{ .Os }}-{{ .Arch }}" + wrap_in_directory: true + replacements: + amd64: 64 + format: tar.gz + format_overrides: + - goos: windows + format: zip + files: + - LICENSE + - README.md + - id: npm + name_template: "{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}" + wrap_in_directory: true + replacements: + amd64: 64 + format: tar.gz + files: + - LICENSE + - src: "scripts/npm-native-packages/build/{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}/package.json" + dst: "workaround/.." + strip_parent: true + - src: "scripts/npm-native-packages/build/{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}/README.md" + dst: "workaround/.." + strip_parent: true + - src: "scripts/npm-native-packages/build/{{ .ProjectName }}-{{ .Os }}-{{ .Arch }}/bin/*" + dst: "bin/" + strip_parent: true +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" +release: + github: + owner: vercel + name: turborepo + ids: + - github + prerelease: auto + disable: true +publishers: + - name: npm + ids: + - npm + cmd: "npm publish{{ if .Prerelease }} --tag canary{{ end }} {{ abs .ArtifactPath }}" diff --git a/cli/cross-release.yml b/cli/cross-release.yml new file mode 100644 index 0000000000000..f5d6fd12a22da --- /dev/null +++ b/cli/cross-release.yml @@ -0,0 +1,53 @@ +project_name: turbo +before: + hooks: + - make compile-protos + - go mod tidy + +dist: dist-cross + +builds: + - id: turbo + main: ./cmd/turbo + binary: bin/turbo + flags: + - -trimpath + ldflags: + - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + env: + - CGO_ENABLED=1 + targets: + - linux_arm64 + - linux_amd64 + - windows_arm64 + - windows_amd64 + overrides: + - goos: linux + goarch: arm64 + env: + - CC=aarch64-linux-gnu-gcc + - CXX=aarch64-linux-gnu-g++ + - goos: linux + goarch: amd64 + goamd64: v1 + env: + - CC=x86_64-linux-gnu-gcc + - CXX=x86_64-linux-gnu-g++ + - goos: windows + goarch: arm64 + env: + - CC=/llvm-mingw/llvm-mingw/bin/aarch64-w64-mingw32-gcc + - CXX=/llvm-mingw/llvm-mingw/bin/aarch64-w64-mingw32-g++ + - goos: windows + goarch: amd64 + goamd64: v1 + env: + - CC=x86_64-w64-mingw32-gcc + - CXX=x86_64-w64-mingw32-g++ + +archives: + - format: binary + +release: + disable: true diff --git a/cli/darwin-release.yml b/cli/darwin-release.yml new file mode 100644 index 0000000000000..9e96aa9ef9bf1 --- /dev/null +++ b/cli/darwin-release.yml @@ -0,0 +1,28 @@ +project_name: turbo +before: + hooks: + - make compile-protos + - go mod tidy + +dist: dist-darwin + +builds: + - id: turbo + main: ./cmd/turbo + binary: bin/turbo + flags: + - -trimpath + ldflags: + - -s -w -X main.version={{.Version}} -X main.commit={{.Commit}} -X main.date={{.CommitDate}} -X main.builtBy=goreleaser + mod_timestamp: "{{ .CommitTimestamp }}" + env: + - CGO_ENABLED=1 + targets: + - darwin_arm64 + - darwin_amd64 + +archives: + - format: binary + +release: + disable: true From 3ccd47e2321514d36bf650002f04fba05411930f Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Mon, 3 Oct 2022 15:55:11 -0700 Subject: [PATCH 2/5] Use vercel build container --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f029b3ce8d81f..b190e4f7bdb91 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: needs: [smoke-test] runs-on: ubuntu-latest container: - image: docker://ghcr.io/gsoltis/turbo-cross:v1.18.5 + image: docker://ghcr.io/vercel/turbo-cross:v1.18.5 steps: - uses: actions/checkout@v3 with: From 75879d1df6428b61893d8022b2087cb49391a4e9 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Mon, 3 Oct 2022 15:58:42 -0700 Subject: [PATCH 3/5] Revert comment in .gitignore --- cli/.gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/.gitignore b/cli/.gitignore index 0175fb3d52d66..6392e24877831 100644 --- a/cli/.gitignore +++ b/cli/.gitignore @@ -1,4 +1,4 @@ -#/internal/turbodprotocol/*.go +/internal/turbodprotocol/*.go /demo/ /dist/ From 12c84a35b8afc27a64d257e7e401e161eab03b61 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Fri, 7 Oct 2022 11:08:52 -0700 Subject: [PATCH 4/5] Add comments about the release process --- .github/workflows/release.yml | 6 ++++++ .github/workflows/stage.yml | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b190e4f7bdb91..b373b823dd3e2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,7 @@ jobs: run: pnpm -- turbo run test --filter=cli --color darwin: + name: "Build Darwin Binares" needs: [smoke-test] runs-on: macos-latest steps: @@ -59,6 +60,7 @@ jobs: # compiles linux and windows in a container cross: + name: "Build Linux and Windows Binaries" needs: [smoke-test] runs-on: ubuntu-latest container: @@ -121,6 +123,10 @@ jobs: env: GORELEASER_KEY: ${{ secrets.GORELEASER_KEY }} + # Download the prebuilt binaries for each platform, + # combine them into a single dist-combined folder, + # then delegate to goreleaser / Makefile for deploy. + # Finally, set up a PR for the branch we released off of - name: Download Cross-compiled Artifacts uses: actions/download-artifact@v3 with: diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 1c03ccd6c6294..8c468270c017e 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -1,6 +1,7 @@ name: Create Release Branch -# TODO: set outputs? optionally trigger release immediately? +# TODO: Once we have confidence with the release process, add an +# input to allow automatically kicking off the downstream release. on: workflow_dispatch: inputs: From b33d84d3a4a286fd1e32b116d5ee275dd8342dc2 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Fri, 7 Oct 2022 11:42:41 -0700 Subject: [PATCH 5/5] Label bump canary option --- .github/workflows/stage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 8c468270c017e..b4f4899789e46 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: inputs: increment: - description: "SemVer Increment" + description: "SemVer Increment (prerelease = bump canary)" required: true default: "prerelease" type: choice