From 231fcf9f992667c5c712cb1cdfa1341decababf4 Mon Sep 17 00:00:00 2001 From: Mehul Kar Date: Wed, 14 Sep 2022 23:54:40 -0700 Subject: [PATCH] Improve CI setup (#1904) Changes the CI setup to: - Extract Go and node setup to a "composite" action, allowing us to setup all CI jobs consistently. For example, the golangci-lint was building turbo with Go 1.17, while the Go unit and e2e tests were building with Go 1.18. - Parallelize some jobs for faster feedback Separates Go unit and e2e tests, and JS lint and tests into separate workflows. The tradeoff is that each Github Action runner will need to install Go and Node and build turbo multiple times, but this is fairly fast, and these steps are highly cacheable. - Improve the names of Workflows, Jobs, and Steps to be more debuggable Some of the naming changes are for consistency, but some make it easier to scan the list of PR checks and the Action summaries to see what failed and where. In some cases, the names of Steps are removed, as the command itself is descriptive enough, that we do not need to name it. Co-authored-by: Nathan Hammond --- .github/actions/setup-go/action.yml | 27 +++ .github/actions/setup-node/action.yml | 40 +++++ .../{large-monorepo.yml => benchmark.yml} | 39 +---- .github/workflows/ci-go.yml | 154 ------------------ .github/workflows/ci-js.yml | 89 ---------- .github/workflows/pr-go-e2e.yml | 31 ++++ .../{golangci-lint.yml => pr-go-lint.yml} | 34 +--- .github/workflows/pr-go-run-examples.yml | 73 +++++++++ .github/workflows/pr-go-unit.yml | 29 ++++ .github/workflows/pr-js-lint.yml | 33 ++++ .github/workflows/pr-js-tests.yml | 33 ++++ .github/workflows/release.yml | 38 +---- 12 files changed, 279 insertions(+), 341 deletions(-) create mode 100644 .github/actions/setup-go/action.yml create mode 100644 .github/actions/setup-node/action.yml rename .github/workflows/{large-monorepo.yml => benchmark.yml} (70%) delete mode 100644 .github/workflows/ci-go.yml delete mode 100644 .github/workflows/ci-js.yml create mode 100644 .github/workflows/pr-go-e2e.yml rename .github/workflows/{golangci-lint.yml => pr-go-lint.yml} (68%) create mode 100644 .github/workflows/pr-go-run-examples.yml create mode 100644 .github/workflows/pr-go-unit.yml create mode 100644 .github/workflows/pr-js-lint.yml create mode 100644 .github/workflows/pr-js-tests.yml diff --git a/.github/actions/setup-go/action.yml b/.github/actions/setup-go/action.yml new file mode 100644 index 0000000000000..889a542141fba --- /dev/null +++ b/.github/actions/setup-go/action.yml @@ -0,0 +1,27 @@ +name: "Turborepo Go Setup" +description: "Sets Go up for CI" +inputs: + github-token: + description: "GitHub token. You can pass secrets.GITHUB_TOKEN" + required: true +runs: + using: "composite" + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.18 + cache: true + cache-dependency-path: cli/go.sum + + - name: Set Up Protoc + uses: arduino/setup-protoc@v1 + with: + version: "3.x" + repo-token: ${{ inputs.github-token }} + + - name: Set Up GRPC protobuf + shell: bash + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 diff --git a/.github/actions/setup-node/action.yml b/.github/actions/setup-node/action.yml new file mode 100644 index 0000000000000..a7143c1c4c022 --- /dev/null +++ b/.github/actions/setup-node/action.yml @@ -0,0 +1,40 @@ +name: "Turborepo Node.js Setup" +description: "Sets Node.js up for CI" +inputs: + enable-corepack: + description: "Control turning on corepack." + required: false + default: true +runs: + using: "composite" + steps: + - name: Setup pnpm + uses: pnpm/action-setup@v2.2.2 + with: + version: 7.2.1 + + - name: Setup Node.js + uses: actions/setup-node@v2 + with: + node-version: 16 + cache: pnpm + + - name: Configure corepack + # Forcibly upgrade our available version of corepack. + # The bundled version in node 16 has known issues. + # Prepends the npm bin dir so that it is always first. + shell: bash + run: | + npm install --force --global corepack@latest + npm config get prefix >> $GITHUB_PATH + corepack enable + + - name: Enable corepack + if: ${{ inputs.enable-corepack == 'true' }} + shell: bash + run: | + corepack enable + + - name: pnpm install + shell: bash + run: pnpm install diff --git a/.github/workflows/large-monorepo.yml b/.github/workflows/benchmark.yml similarity index 70% rename from .github/workflows/large-monorepo.yml rename to .github/workflows/benchmark.yml index f52f55a406f50..fda33da2b39e7 100644 --- a/.github/workflows/large-monorepo.yml +++ b/.github/workflows/benchmark.yml @@ -17,46 +17,15 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - - name: Check out code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-node + - uses: ./.github/actions/setup-go with: - fetch-depth: 2 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - id: go - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 - - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Setup Node.js environment - uses: actions/setup-node@v2 - with: - node-version: 16 - cache: pnpm + github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Build run: cd cli && make turbo - - name: Install dependencies - run: pnpm install --filter=benchmark - - name: Download previous benchmark results # continue on error so that we handle the bootstrap case where there is no previous data continue-on-error: true diff --git a/.github/workflows/ci-go.yml b/.github/workflows/ci-go.yml deleted file mode 100644 index 527e1f64254de..0000000000000 --- a/.github/workflows/ci-go.yml +++ /dev/null @@ -1,154 +0,0 @@ -name: CI Go - -env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: ["main"] - pull_request: - types: [opened, edited, synchronize] - -jobs: - build: - name: build and test - timeout-minutes: 15 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 - - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Setup Node.js environment - uses: actions/setup-node@v2 - with: - node-version: 16 - cache: pnpm - - - name: Configure corepack - # Forcibly upgrade our available version of corepack. - # The bundled version in node 16 has known issues. - # Prepends the corepack bin dir so that it is always first. - shell: bash - run: | - npm install --force --global corepack@latest - npm config get prefix >> $GITHUB_PATH - corepack enable - - - name: Install dependencies - run: pnpm install - - - name: Build & Unit Test - run: pnpm -- turbo run test --filter=cli --color - - - name: E2E Tests - run: pnpm -- turbo run e2e --filter=cli - - examples: - name: run examples - timeout-minutes: 15 - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - manager: [yarn, npm] - example: [basic, design-system, kitchen-sink] - include: - - os: ubuntu-latest - manager: pnpm - example: with-pnpm - - os: macos-latest - manager: pnpm - example: with-pnpm - - runs-on: ${{ matrix.os }} - steps: - - name: Install Sponge - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - sudo apt-get install -y moreutils - else - brew install moreutils - fi - - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Set up Go 1 - uses: actions/setup-go@v3 - with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 - - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Make sure pnpm always has a cache - shell: bash - run: | - mkdir -p `pnpm store path` - - - name: Setup Node.js environment - uses: actions/setup-node@v2 - with: - node-version: 16 - cache: ${{matrix.manager}} - cache-dependency-path: package.json - - - name: Check ${{matrix.example}} example with ${{ matrix.manager }} - shell: bash - env: - FORCE_COLOR: true - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ secrets.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - run: pnpm -- turbo run run-example -- "${{ matrix.example }}" "${{ matrix.manager }}" diff --git a/.github/workflows/ci-js.yml b/.github/workflows/ci-js.yml deleted file mode 100644 index 3e287503514d0..0000000000000 --- a/.github/workflows/ci-js.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: CI JS - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - -on: - push: - branches: ["main"] - pull_request: - types: [opened, edited, synchronize] - -jobs: - build: - name: build and test - timeout-minutes: 30 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - env: - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ secrets.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 - - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Setup Node.js environment - uses: actions/setup-node@v2 - with: - node-version: 16 - cache: pnpm - - - name: Configure corepack - # Forcibly upgrade our available version of corepack. - # The bundled version in node 16 has known issues. - # Prepends the corepack bin dir so that it is always first. - shell: bash - run: | - npm install --force --global corepack@latest - npm config get prefix >> $GITHUB_PATH - corepack enable - - - name: Install dependencies - run: pnpm install - - - name: Lint - run: pnpm -- turbo run lint --filter=!cli - - - name: Check types - run: pnpm -- turbo run check-types --filter=!cli - - - name: Run packages tests - run: pnpm -- turbo run test --filter=!create-turbo --filter=!cli --color - - - name: Run create-turbo tests - # `pnpm --` calls the `turbo` script in the root package.json - # --filter and --color args are for turbo CLI - run: pnpm -- turbo run test --filter=create-turbo --color diff --git a/.github/workflows/pr-go-e2e.yml b/.github/workflows/pr-go-e2e.yml new file mode 100644 index 0000000000000..52f2e33f3f03d --- /dev/null +++ b/.github/workflows/pr-go-e2e.yml @@ -0,0 +1,31 @@ +name: CLI E2E tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + test: + timeout-minutes: 15 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-node + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: E2E Tests + run: pnpm -- turbo run e2e --filter=cli + diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/pr-go-lint.yml similarity index 68% rename from .github/workflows/golangci-lint.yml rename to .github/workflows/pr-go-lint.yml index deccf00472db0..e66114cb234a8 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/pr-go-lint.yml @@ -1,19 +1,10 @@ -name: golangci-lint +name: CLI Linter concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - on: - # push: - # tags: - # - v* - # branches: - # - master - # - main pull_request: types: [opened, edited, synchronize] @@ -22,30 +13,13 @@ permissions: # Optional: allow read access to pull request. Use with `only-new-issues` option. pull-requests: read jobs: - golangci: - name: lint + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-go with: - fetch-depth: 2 - - - uses: actions/setup-go@v3 - with: - go-version: 1.17 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 + github-token: "${{ secrets.GITHUB_TOKEN }}" - name: Setup Protos run: cd cli && make compile-protos diff --git a/.github/workflows/pr-go-run-examples.yml b/.github/workflows/pr-go-run-examples.yml new file mode 100644 index 0000000000000..34f291aaeee11 --- /dev/null +++ b/.github/workflows/pr-go-run-examples.yml @@ -0,0 +1,73 @@ +name: CLI Run Examples + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + examples: + name: CLI run examples + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + manager: [yarn, npm] + example: [basic, design-system, kitchen-sink] + include: + - os: ubuntu-latest + manager: pnpm + example: with-pnpm + - os: macos-latest + manager: pnpm + example: with-pnpm + + runs-on: ${{ matrix.os }} + steps: + # Used by scripts/check-examples.sh + - name: Install Sponge + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install -y moreutils + else + brew install moreutils + fi + + - uses: actions/checkout@v3 + + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - uses: pnpm/action-setup@v2.2.2 + with: + version: 7.2.1 + + - name: Make sure pnpm always has a cache + shell: bash + run: | + mkdir -p `pnpm store path` + + - name: Setup Node.js environment + uses: actions/setup-node@v2 + with: + node-version: 16 + cache: ${{matrix.manager}} + cache-dependency-path: package.json + + - name: Check \"${{matrix.example}}\" example with \"${{ matrix.manager }}\" + shell: bash + env: + FORCE_COLOR: true + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + run: pnpm -- turbo run run-example -- "${{ matrix.example }}" "${{ matrix.manager }}" diff --git a/.github/workflows/pr-go-unit.yml b/.github/workflows/pr-go-unit.yml new file mode 100644 index 0000000000000..6e44112bf845f --- /dev/null +++ b/.github/workflows/pr-go-unit.yml @@ -0,0 +1,29 @@ +name: CLI Unit Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + test: + timeout-minutes: 15 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-node + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - run: pnpm -- turbo run test --filter=cli --color diff --git a/.github/workflows/pr-js-lint.yml b/.github/workflows/pr-js-lint.yml new file mode 100644 index 0000000000000..f93dc7bdfb5ec --- /dev/null +++ b/.github/workflows/pr-js-lint.yml @@ -0,0 +1,33 @@ +name: JS Package Linter + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + lint: + timeout-minutes: 30 + runs-on: ubuntu-latest + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-node + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Lint + run: pnpm -- turbo run lint --filter=!cli + + - name: Check types + run: pnpm -- turbo run check-types --filter=!cli diff --git a/.github/workflows/pr-js-tests.yml b/.github/workflows/pr-js-tests.yml new file mode 100644 index 0000000000000..815319fc67624 --- /dev/null +++ b/.github/workflows/pr-js-tests.yml @@ -0,0 +1,33 @@ +name: JS Package Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + test: + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/setup-node + - uses: ./.github/actions/setup-go + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - run: pnpm -- turbo run test --filter=!cli --color diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0e7dcab52cd7..36200759990fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,29 +31,15 @@ jobs: build: runs-on: macos-latest steps: - - name: Check out code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: - fetch-depth: 2 token: ${{ secrets.TURBOBOT }} - - - name: Set up Go - uses: actions/setup-go@v3 + - uses: ./.github/actions/setup-node with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 + enable-corepack: false + - uses: ./.github/actions/setup-go with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 + github-token: "${{ secrets.GITHUB_TOKEN }}" - name: golangci-lint uses: golangci/golangci-lint-action@v3 @@ -68,20 +54,6 @@ jobs: # We ignore the output of this run, instead using it just to install the binary. args: --issues-exit-code=0 - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Setup Node.js environment - uses: actions/setup-node@v3 - with: - node-version: 16 - registry-url: 'https://registry.npmjs.org' - cache: pnpm - - - name: Install dependencies - run: pnpm install --filter=turbo-monorepo - - name: Configure git run: | git config --global user.name 'Turbobot'