From 789b26de3b4afc8888d36bba8acae70cf43a36e6 Mon Sep 17 00:00:00 2001 From: Mehul Kar Date: Thu, 8 Sep 2022 13:49:55 -0700 Subject: [PATCH] Improve CI setup Changes the CI setup to: - Extract Go and node setup to a "composite" action, allowing us to setup all CI jobs consistently. For example, the golangci-lint was building turbo with Go 1.17, while the Go unit and e2e tests were building with Go 1.18. - Parallelize some jobs for faster feedback Separates Go unit and e2e tests, and JS lint and tests into separate workflows. The tradeoff is that each Github Action runner will need to install Go and Node and build turbo multiple times, but this is fairly fast, and these steps are highly cacheable. - Improve the names of Workflows, Jobs, and Steps to be more debuggable Some of the naming changes are for consistency, but some make it easier to scan the list of PR checks and the Action summaries to see what failed and where. In some cases, the names of Steps are removed, as the command itself is descriptive enough, that we do not need to name it. --- .github/actions/setup-deps/action.yml | 60 +++++++ .../{large-monorepo.yml => benchmark.yml} | 23 +-- .github/workflows/ci-go.yml | 154 ------------------ .github/workflows/ci-js.yml | 89 ---------- .github/workflows/pr-go-e2e.yml | 36 ++++ .../{golangci-lint.yml => pr-go-lint.yml} | 29 +--- .github/workflows/pr-go-run-examples.yml | 80 +++++++++ .github/workflows/pr-go-unit.yml | 34 ++++ .github/workflows/pr-js-lint.yml | 42 +++++ .github/workflows/pr-js-tests.yml | 38 +++++ .github/workflows/release.yml | 22 +-- 11 files changed, 303 insertions(+), 304 deletions(-) create mode 100644 .github/actions/setup-deps/action.yml rename .github/workflows/{large-monorepo.yml => benchmark.yml} (80%) delete mode 100644 .github/workflows/ci-go.yml delete mode 100644 .github/workflows/ci-js.yml create mode 100644 .github/workflows/pr-go-e2e.yml rename .github/workflows/{golangci-lint.yml => pr-go-lint.yml} (72%) create mode 100644 .github/workflows/pr-go-run-examples.yml create mode 100644 .github/workflows/pr-go-unit.yml create mode 100644 .github/workflows/pr-js-lint.yml create mode 100644 .github/workflows/pr-js-tests.yml diff --git a/.github/actions/setup-deps/action.yml b/.github/actions/setup-deps/action.yml new file mode 100644 index 0000000000000..35b1514081ac9 --- /dev/null +++ b/.github/actions/setup-deps/action.yml @@ -0,0 +1,60 @@ +name: "Turborepo CI Setup" +description: "Sets up everything for CI" +inputs: + github-token: + description: "Github token. You can pass secrets.GITHUB_TOKEN" + required: true + setup-node: + description: "Set to false to prevent Node setup steps" + required: false + default: true +runs: + using: "composite" + steps: + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.18.0 + cache: true + cache-dependency-path: cli/go.sum + + - name: Set Up Protoc + uses: arduino/setup-protoc@v1 + with: + version: "3.x" + repo-token: ${{ inputs.github-token }} + + - name: Set Up GRPC protobuf + shell: bash + run: | + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 + go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 + + # Setup Node things + - name: Setup pnpm + uses: pnpm/action-setup@v2.2.2 + if: ${{ inputs.setup-node == 'true' }} + with: + version: 7.2.1 + + - name: Setup Node.js + uses: actions/setup-node@v2 + if: ${{ inputs.setup-node == 'true' }} + with: + node-version: 16 + cache: pnpm + + - name: Configure corepack + if: ${{ inputs.setup-node == 'true' }} + # Forcibly upgrade our available version of corepack. + # The bundled version in node 16 has known issues. + # Prepends the corepack bin dir so that it is always first. + shell: bash + run: | + npm install --force --global corepack@latest + npm config get prefix >> $GITHUB_PATH + corepack enable + + - run: pnpm install + shell: bash + if: ${{ inputs.setup-node == 'true' }} diff --git a/.github/workflows/large-monorepo.yml b/.github/workflows/benchmark.yml similarity index 80% rename from .github/workflows/large-monorepo.yml rename to .github/workflows/benchmark.yml index f52f55a406f50..f669d4472760a 100644 --- a/.github/workflows/large-monorepo.yml +++ b/.github/workflows/benchmark.yml @@ -17,29 +17,14 @@ jobs: os: [ubuntu-latest, macos-latest] steps: - - name: Check out code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 2 - - name: Set up Go - uses: actions/setup-go@v3 + - uses: ./.github/actions/setup-deps with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - id: go - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 + github-token: "${{ secrets.GITHUB_TOKEN }}" + setup-node: false # TODO: does this need to be customized in this workflow? - uses: pnpm/action-setup@v2.2.2 with: diff --git a/.github/workflows/ci-go.yml b/.github/workflows/ci-go.yml deleted file mode 100644 index 527e1f64254de..0000000000000 --- a/.github/workflows/ci-go.yml +++ /dev/null @@ -1,154 +0,0 @@ -name: CI Go - -env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: ["main"] - pull_request: - types: [opened, edited, synchronize] - -jobs: - build: - name: build and test - timeout-minutes: 15 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 - - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Setup Node.js environment - uses: actions/setup-node@v2 - with: - node-version: 16 - cache: pnpm - - - name: Configure corepack - # Forcibly upgrade our available version of corepack. - # The bundled version in node 16 has known issues. - # Prepends the corepack bin dir so that it is always first. - shell: bash - run: | - npm install --force --global corepack@latest - npm config get prefix >> $GITHUB_PATH - corepack enable - - - name: Install dependencies - run: pnpm install - - - name: Build & Unit Test - run: pnpm -- turbo run test --filter=cli --color - - - name: E2E Tests - run: pnpm -- turbo run e2e --filter=cli - - examples: - name: run examples - timeout-minutes: 15 - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - manager: [yarn, npm] - example: [basic, design-system, kitchen-sink] - include: - - os: ubuntu-latest - manager: pnpm - example: with-pnpm - - os: macos-latest - manager: pnpm - example: with-pnpm - - runs-on: ${{ matrix.os }} - steps: - - name: Install Sponge - shell: bash - run: | - if [ "$RUNNER_OS" == "Linux" ]; then - sudo apt-get install -y moreutils - else - brew install moreutils - fi - - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Set up Go 1 - uses: actions/setup-go@v3 - with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 - - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Make sure pnpm always has a cache - shell: bash - run: | - mkdir -p `pnpm store path` - - - name: Setup Node.js environment - uses: actions/setup-node@v2 - with: - node-version: 16 - cache: ${{matrix.manager}} - cache-dependency-path: package.json - - - name: Check ${{matrix.example}} example with ${{ matrix.manager }} - shell: bash - env: - FORCE_COLOR: true - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ secrets.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - run: pnpm -- turbo run run-example -- "${{ matrix.example }}" "${{ matrix.manager }}" diff --git a/.github/workflows/ci-js.yml b/.github/workflows/ci-js.yml deleted file mode 100644 index 3e287503514d0..0000000000000 --- a/.github/workflows/ci-js.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: CI JS - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - -on: - push: - branches: ["main"] - pull_request: - types: [opened, edited, synchronize] - -jobs: - build: - name: build and test - timeout-minutes: 30 - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest, windows-latest] - env: - TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} - TURBO_TEAM: ${{ secrets.TURBO_TEAM }} - TURBO_REMOTE_ONLY: true - - steps: - - name: Check out code - uses: actions/checkout@v3 - with: - fetch-depth: 2 - - - name: Set up Go - uses: actions/setup-go@v3 - with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 - - - uses: pnpm/action-setup@v2.2.2 - with: - version: 7.2.1 - - - name: Setup Node.js environment - uses: actions/setup-node@v2 - with: - node-version: 16 - cache: pnpm - - - name: Configure corepack - # Forcibly upgrade our available version of corepack. - # The bundled version in node 16 has known issues. - # Prepends the corepack bin dir so that it is always first. - shell: bash - run: | - npm install --force --global corepack@latest - npm config get prefix >> $GITHUB_PATH - corepack enable - - - name: Install dependencies - run: pnpm install - - - name: Lint - run: pnpm -- turbo run lint --filter=!cli - - - name: Check types - run: pnpm -- turbo run check-types --filter=!cli - - - name: Run packages tests - run: pnpm -- turbo run test --filter=!create-turbo --filter=!cli --color - - - name: Run create-turbo tests - # `pnpm --` calls the `turbo` script in the root package.json - # --filter and --color args are for turbo CLI - run: pnpm -- turbo run test --filter=create-turbo --color diff --git a/.github/workflows/pr-go-e2e.yml b/.github/workflows/pr-go-e2e.yml new file mode 100644 index 0000000000000..d29fa7cb395ba --- /dev/null +++ b/.github/workflows/pr-go-e2e.yml @@ -0,0 +1,36 @@ +name: CLI E2E tests + +env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + test: + timeout-minutes: 15 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: ./.github/actions/setup-deps + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: E2E Tests + run: pnpm -- turbo run e2e --filter=cli + diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/pr-go-lint.yml similarity index 72% rename from .github/workflows/golangci-lint.yml rename to .github/workflows/pr-go-lint.yml index deccf00472db0..eb70707953af3 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/pr-go-lint.yml @@ -1,4 +1,4 @@ -name: golangci-lint +name: CLI Linter concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -8,12 +8,6 @@ env: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} on: - # push: - # tags: - # - v* - # branches: - # - master - # - main pull_request: types: [opened, edited, synchronize] @@ -22,30 +16,17 @@ permissions: # Optional: allow read access to pull request. Use with `only-new-issues` option. pull-requests: read jobs: - golangci: - name: lint + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 2 - - uses: actions/setup-go@v3 + - uses: ./.github/actions/setup-deps with: - go-version: 1.17 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 + github-token: "${{ secrets.GITHUB_TOKEN }}" + setup-node: false - name: Setup Protos run: cd cli && make compile-protos diff --git a/.github/workflows/pr-go-run-examples.yml b/.github/workflows/pr-go-run-examples.yml new file mode 100644 index 0000000000000..4b74d763eba7c --- /dev/null +++ b/.github/workflows/pr-go-run-examples.yml @@ -0,0 +1,80 @@ +name: CLI Run Examples + +env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + examples: + name: CLI run examples + timeout-minutes: 15 + + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + manager: [yarn, npm] + example: [basic, design-system, kitchen-sink] + include: + - os: ubuntu-latest + manager: pnpm + example: with-pnpm + - os: macos-latest + manager: pnpm + example: with-pnpm + + runs-on: ${{ matrix.os }} + steps: + # TODO: what is this for? The other Go workflows don't need it + - name: Install Sponge + shell: bash + run: | + if [ "$RUNNER_OS" == "Linux" ]; then + sudo apt-get install -y moreutils + else + brew install moreutils + fi + + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: ./.github/actions/setup-deps + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + setup-node: false # TODO: Does this need to be customized in this workflow? + + - uses: pnpm/action-setup@v2.2.2 + with: + version: 7.2.1 + + # TODO: Is this necessary? The other Go jobs don't include this + - name: Make sure pnpm always has a cache + shell: bash + run: | + mkdir -p `pnpm store path` + + - name: Setup Node.js environment + uses: actions/setup-node@v2 + with: + node-version: 16 + cache: ${{matrix.manager}} + cache-dependency-path: package.json + + - name: Check \"${{matrix.example}}\" example with \"${{ matrix.manager }}\" + shell: bash + env: + FORCE_COLOR: true + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + run: pnpm -- turbo run run-example -- "${{ matrix.example }}" "${{ matrix.manager }}" diff --git a/.github/workflows/pr-go-unit.yml b/.github/workflows/pr-go-unit.yml new file mode 100644 index 0000000000000..9b7305f313a17 --- /dev/null +++ b/.github/workflows/pr-go-unit.yml @@ -0,0 +1,34 @@ +name: CLI Unit Tests + +env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + test: + timeout-minutes: 15 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: ./.github/actions/setup-deps + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - run: pnpm -- turbo run test --filter=cli --color diff --git a/.github/workflows/pr-js-lint.yml b/.github/workflows/pr-js-lint.yml new file mode 100644 index 0000000000000..6a2ff9e63e9ce --- /dev/null +++ b/.github/workflows/pr-js-lint.yml @@ -0,0 +1,42 @@ +name: JS Package Linter + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + lint: + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: ./.github/actions/setup-deps + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - name: Lint + run: pnpm -- turbo run lint --filter=!cli + + - name: Check types + run: pnpm -- turbo run check-types --filter=!cli diff --git a/.github/workflows/pr-js-tests.yml b/.github/workflows/pr-js-tests.yml new file mode 100644 index 0000000000000..4bb0c7e9da7e4 --- /dev/null +++ b/.github/workflows/pr-js-tests.yml @@ -0,0 +1,38 @@ +name: JS Package Tests + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + +on: + push: + branches: ["main"] + pull_request: + types: [opened, edited, synchronize] + +jobs: + test: + timeout-minutes: 30 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + TURBO_REMOTE_ONLY: true + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 2 + + - uses: ./.github/actions/setup-deps + with: + github-token: "${{ secrets.GITHUB_TOKEN }}" + + - run: pnpm -- turbo run test --filter=!cli --color diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0e7dcab52cd7..b94853414eace 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,29 +31,15 @@ jobs: build: runs-on: macos-latest steps: - - name: Check out code - uses: actions/checkout@v3 + - uses: actions/checkout@v3 with: fetch-depth: 2 token: ${{ secrets.TURBOBOT }} - - name: Set up Go - uses: actions/setup-go@v3 + - uses: ./.github/actions/setup-deps with: - go-version: 1.18.0 - cache: true - cache-dependency-path: cli/go.sum - - - name: Set Up Protoc - uses: arduino/setup-protoc@v1 - with: - version: "3.x" - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Set Up Go and GRPC protobuf - run: | - go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.0 - go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.2.0 + github-token: "${{ secrets.GITHUB_TOKEN }}" + setup-node: false # TODO: does this need to be customized? (corepack step is omitted here) - name: golangci-lint uses: golangci/golangci-lint-action@v3