chore: optimize CI caching (#2183)

ipfs · Dec 2, 2023 · b8a71d9 · b8a71d9
1 parent 951c7bd
commit b8a71d9
Show file tree

Hide file tree

Showing 10 changed files with 114 additions and 163 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -9,19 +9,10 @@ jobs:
       - uses: actions/checkout@v3.6.0
 
       - name: Setup node
-        uses: actions/setup-node@v3.6.0
+        uses: actions/setup-node@v4
         with:
           node-version: 18.14.0
-
-      - name: Cache bigger downloads
-        uses: actions/cache@v3.3.1
-        id: cache
-        with:
-          path: ${{ github.workspace }}/.cache
-          key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-            ${{ runner.os }}-
+          cache: 'npm'
 
       # Separate cache for build dir, we reuse it in release publish workflow
       - name: Cache build output
@@ -37,7 +28,7 @@ jobs:
       - name: Install dependencies
         # We only need to install deps if build isn't cached.
         if: steps.build-cache.outputs.cache-hit != 'true'
-        run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
+        run: npm ci --prefer-offline --no-audit --progress=false
 
       - name: Confirm build works
         if: steps.build-cache.outputs.cache-hit != 'true'
@@ -47,6 +38,8 @@ jobs:
       # - this is not for releases, but for quick testing during the dev
       # - action artifacts can be downloaded for 90 days, then are removed by github
       - name: Attach produced build to Github Action
+        # only re-upload if we re-built
+        if: steps.build-cache.outputs.cache-hit != 'true'
         uses: actions/upload-artifact@v3
         with:
           name: ipfs-webui_${{ github.sha }}-${{ runner.os }}-build

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -33,15 +33,20 @@ jobs:
       - uses: actions/checkout@v3.6.0
 
       - name: Setup node
-        uses: actions/setup-node@v3.6.0
+        uses: actions/setup-node@v4
         with:
           node-version: 18.14.0
+          cache: 'npm'
 
-      - name: Download build artifact
-        uses: actions/download-artifact@v3
+      - name: Cache build output
+        uses: actions/cache@v3.3.1
+        id: build-cache
         with:
-          name: ipfs-webui_${{ github.sha }}-${{ runner.os }}-build
           path: build
+          # Build hash should be based on runner.os, src files, and dependencies used.
+          key: ${{ runner.os }}-build-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
+          restore-keys: |
+            ${{ runner.os }}-build-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
 
       - name: Cache bigger downloads
         uses: actions/cache@v3.3.1
@@ -232,12 +237,13 @@ jobs:
           echo ${{ needs.publishPreview.outputs.cid }} > .cid
 
       - name: Setup node
-        uses: actions/setup-node@v3.6.0
+        uses: actions/setup-node@v4
         with:
           node-version: 18.14.0
+          cache: 'npm'
 
       - name: Install dependencies
-        run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
+        run: npm ci --prefer-offline --no-audit --progress=false
 
       - name: Download CAR artifact
         uses: actions/download-artifact@v3

diff --git a/.github/workflows/eslint.yml b/.github/workflows/eslint.yml
@@ -9,22 +9,13 @@ jobs:
       - uses: actions/checkout@v3.6.0
 
       - name: Setup node
-        uses: actions/setup-node@v3.6.0
+        uses: actions/setup-node@v4
         with:
           node-version: 18.14.0
-
-      - name: Cache bigger downloads
-        uses: actions/cache@v3.3.1
-        id: cache
-        with:
-          path: ${{ github.workspace }}/.cache
-          key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-            ${{ runner.os }}-
+          cache: 'npm'
 
       - name: Install dependencies
-        run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
+        run: npm ci --prefer-offline --no-audit --progress=false
 
       - name: ESLint
         run: npm run eslint
diff --git a/.github/workflows/node-test.yml b/.github/workflows/node-test.yml
@@ -19,12 +19,12 @@ jobs:
         if: github.ref == 'refs/heads/main'
         uses: RubbaBoy/BYOB@24f464284c1fd32028524b59607d417a2e36fee7
         with:
-          ICON: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
-          NAME: "node-${{ inputs.gh-node-version }}"
-          LABEL: "${{ inputs.gh-node-version }}"
-          STATUS: "Building..."
-          COLOR: grey
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          icon: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
+          name: "node-${{ inputs.gh-node-version }}"
+          label: "${{ inputs.gh-node-version }}"
+          status: "Building..."
+          color: grey
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
   test-node:
     needs: [reset-badge]
@@ -40,9 +40,10 @@ jobs:
     - uses: actions/checkout@v3.6.0
 
     - name: Use Node.js ${{ inputs.gh-node-version }}
-      uses: actions/setup-node@v3.6.0
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ inputs.gh-node-version }}
+        cache: 'npm'
 
     - name: Remove .npmrc to allow non-strict engines
       run: |
@@ -53,22 +54,12 @@ jobs:
       run: |
         echo "version=$(node --version)" >> $GITHUB_OUTPUT
 
-    - name: Cache bigger downloads
-      uses: actions/cache@v3.3.1
-      id: cache
-      with:
-        path: ${{ github.workspace }}/.cache
-        key: ${{ runner.os }}-${{ inputs.gh-node-version }}-${{ hashFiles('package.json', 'package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-${{ inputs.gh-node-version }}-${{ hashFiles('package.json', 'package-lock.json') }}
-          ${{ runner.os }}-${{ inputs.gh-node-version }}
-
     - name: Install node-pre-gyp
       run: |
-        npm install node-pre-gyp --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
+        npm install node-pre-gyp --prefer-offline --no-audit --progress=false
 
     - name: Install dependencies
-      run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
+      run: npm install --progress=false
 
     - name: Confirm build works with Node.js ${{ inputs.gh-node-version }} on ${{ matrix.os }}
       run: npm run build
@@ -82,20 +73,20 @@ jobs:
         if: ${{ needs.test-node.result == 'success' && github.ref == 'refs/heads/main' }}
         uses: RubbaBoy/BYOB@24f464284c1fd32028524b59607d417a2e36fee7
         with:
-          ICON: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
-          NAME: "node-${{ inputs.gh-node-version }}"
-          LABEL: "${{ needs.test-node.outputs.node-version }}"
-          STATUS: ${{needs.test-node.result}}
-          COLOR: green
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          icon: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
+          name: "node-${{ inputs.gh-node-version }}"
+          label: "${{ needs.test-node.outputs.node-version }}"
+          status: ${{needs.test-node.result}}
+          color: green
+          github_token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Create failure badge
         if: ${{ needs.test-node.result != 'success' && github.ref == 'refs/heads/main' }}
         uses: RubbaBoy/BYOB@24f464284c1fd32028524b59607d417a2e36fee7
         with:
-          ICON: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
-          NAME: "node-${{ inputs.gh-node-version }}"
-          LABEL: "${{ needs.test-node.outputs.node-version }}"
-          STATUS: ${{needs.test-node.result}}
-          COLOR: red
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          icon: https://raw.githubusercontent.com/devicons/devicon/master/icons/nodejs/nodejs-original.svg
+          name: "node-${{ inputs.gh-node-version }}"
+          label: "${{ needs.test-node.outputs.node-version }}"
+          status: ${{needs.test-node.result}}
+          color: red
+          github_token: ${{ secrets.GITHUB_TOKEN }}
diff --git a/.github/workflows/node-versions.yml b/.github/workflows/node-versions.yml
@@ -1,4 +1,4 @@
-name: Test build on different node versions on all platforms
+name: Test nodejs support for build on major platforms
 
 on:
   schedule:
@@ -8,26 +8,20 @@ on:
 
 jobs:
 
-  test-node-12:
-    uses: ./.github/workflows/node-test.yml
-    secrets: inherit
-    with:
-      gh-node-version: '12.x'
-
-  test-node-14:
+  test-node-18:
     uses: ./.github/workflows/node-test.yml
     secrets: inherit
     with:
-      gh-node-version: '14.x'
+      gh-node-version: '18.x'
 
-  test-node-16:
+  test-node-20:
     uses: ./.github/workflows/node-test.yml
     secrets: inherit
     with:
-      gh-node-version: '16.x'
+      gh-node-version: '20.x'
 
-  test-node-18:
+  test-node-21:
     uses: ./.github/workflows/node-test.yml
     secrets: inherit
     with:
-      gh-node-version: '18.x'
+      gh-node-version: '21.x'
diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml
@@ -9,31 +9,32 @@ jobs:
       - uses: actions/checkout@v3.6.0
 
       - name: Setup node
-        uses: actions/setup-node@v3.6.0
+        uses: actions/setup-node@v4
         with:
           node-version: 18.14.0
+          cache: 'npm'
 
-      - name: Cache bigger downloads
+      - name: Cache test-build output
         uses: actions/cache@v3.3.1
-        id: cache
+        id: test-build-cache
         with:
-          path: ${{ github.workspace }}/.cache
-          key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
+          path: build
+          key: test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
           restore-keys: |
-            ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-            ${{ runner.os }}-
-
-      - name: Install dependencies
-        run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
+            test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
 
-      - name: Cache test-build output
+      - name: Cache node_modules
         uses: actions/cache@v3.3.1
-        id: test-build-cache
+        id: test-npm-cache
         with:
-          path: build
-          key: test-build-${{ runner.os }}-${{ github.sha }}
+          path: node_modules
+          key: test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
           restore-keys: |
-            test-build-${{ runner.os }}-${{ github.sha }}
+            test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
+
+      - name: Install dependencies
+        if: steps.test-npm-cache.outputs.cache-hit != 'true'
+        run: npm ci --prefer-offline --no-audit --progress=false
 
       # This is required to ensure that our code is instrumented with coverage details
       - name: Run test build
@@ -53,39 +54,43 @@ jobs:
       - uses: actions/checkout@v3.6.0
 
       - name: Setup node
-        uses: actions/setup-node@v3.6.0
+        uses: actions/setup-node@v4
         with:
           node-version: 18.14.0
+          cache: 'npm'
 
-      - name: Cache bigger downloads
+      - name: Cache test-build output
         uses: actions/cache@v3.3.1
-        id: cache
+        id: test-build-cache
         with:
-          path: ${{ github.workspace }}/.cache
-          key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
+          path: build
+          key: test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
           restore-keys: |
-            ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-            ${{ runner.os }}-
+            test-build-${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', '*.js', '*.ts') }}
 
-      - name: Install dependencies
-        run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
-
-      - name: Install playwright browsers
-        run: npx playwright install --with-deps
-
-      - name: Cache test-build output
+      # We don't want to re-install dependencies for every sharded run
+      - name: Cache node_modules
         uses: actions/cache@v3.3.1
-        id: test-build-cache
+        id: test-npm-cache
         with:
-          path: build
-          key: test-build-${{ runner.os }}-${{ github.sha }}
+          path: node_modules
+          key: test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
           restore-keys: |
-            test-build-${{ runner.os }}-${{ github.sha }}
+            test-npm-${{ runner.os }}-${{ github.sha }}-${{ hashFiles('package.json', 'package-lock.json', 'public', 'src/**', 'patches/**', '*.js', '*.ts') }}
+      - name: Install dependencies
+        if: steps.test-npm-cache.outputs.cache-hit != 'true'
+        run: npm ci --prefer-offline --no-audit --progress=false
 
-      # This is required to ensure that our code is instrumented with coverage details
-      - name: Run test build
-        if: steps.test-build-cache.outputs.cache-hit != 'true'
-        run: npm run test:build
+      # Cache playwright binaries
+      - uses: actions/cache@v3
+        id: playwright-cache
+        with:
+          path: |
+            ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ hashFiles('**/package-lock.json') }}
+      - run: npm ci
+      - run: npx playwright install --with-deps
+        if: steps.playwright-cache.outputs.cache-hit != 'true'
 
       - name: Cache nyc_output dir
         uses: actions/cache@v3.3.1
@@ -104,6 +109,14 @@ jobs:
         if: github.ref == 'refs/heads/main' # run non-duplicated tests on non-PRs
         run: npm run test:e2e -- --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }}
 
+      - name: Store Artifacts from Failed Tests
+        if: failure()
+        uses: actions/upload-artifact@v3
+        with:
+          name: test-results
+          path: test-results/
+          retention-days: 7
+
   e2e-coverage: # since we run coverage in shards and some files may not contain coverage, we cache the .nyc_output directory
     name: 'e2e-coverage'
     runs-on: ubuntu-latest
@@ -112,23 +125,10 @@ jobs:
       - uses: actions/checkout@v3.6.0
 
       - name: Setup node
-        uses: actions/setup-node@v3.6.0
+        uses: actions/setup-node@v4
         with:
           node-version: 18.14.0
-
-      - name: Cache bigger downloads
-        uses: actions/cache@v3.3.1
-        id: cache
-        with:
-          path: ${{ github.workspace }}/.cache
-          key: ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ hashFiles('package.json', 'package-lock.json') }}
-            ${{ runner.os }}-
-
-
-      - name: Install dependencies
-        run: npm ci --prefer-offline --no-audit --progress=false --cache ${{ github.workspace }}/.cache/npm
+          cache: 'npm'
 
       - name: Cache nyc_output dir
         uses: actions/cache@v3.3.1