Add sccache to pipeline (#335)

* Add sccache to pipeline * Fix Github Actions build-and-test include path CI checks should correctly run if the build-and-test.yml definition changes now. * Fix sccache installation on Linux, Windows * Debug Windows sccache path * Add cache steps to all build CI jobs Fix cache restore keys. * Fix sccache restore keys
iotaledger · Jul 27, 2021 · 40790f6 · 40790f6
1 parent d82ea1a
commit 40790f6
Showing 1 changed file with 233 additions and 37 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -10,17 +10,35 @@ on:
       - main
       - dev
     paths:
-      - '.github/workflows/format.yml'
+      - '.github/workflows/build-and-test.yml'
       - '**.rs'
       - '**.toml'
 
+env:
+  RUST_BACKTRACE: full
+  RUSTC_WRAPPER: sccache
+  SCCACHE_DOWNLOAD_LINK: https://github.com/mozilla/sccache/releases/download
+  SCCACHE_VERSION: v0.2.15
+  SCCACHE_CACHE_SIZE: 2G
+  SCCACHE_IDLE_TIMEOUT: 0
+  # SCCACHE_RECACHE: 1 # uncomment to clear sccache cache, then re-comment
+
 jobs:
   build-and-test:
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
+        include:
+          - os: ubuntu-latest
+            sccache-path: /home/runner/.cache/sccache
+          - os: macos-latest
+            sccache-path: /Users/runner/Library/Caches/Mozilla.sccache
+          - os: windows-latest
+            sccache-path: C:\\Users\\runner\\AppData\\Local\\Mozilla\\sccache\\cache
+    env:
+      SCCACHE_DIR: ${{ matrix.sccache-path }}
 
     steps:
       - uses: actions/checkout@v2
@@ -39,55 +57,78 @@ jobs:
         if: matrix.os == 'windows-latest'
         run: echo "CURRENT_DATE=$(Get-Date -Format "yyyy-MM-dd")" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
 
-      - name: Cache cargo registry
+      - name: Cache cargo
         uses: actions/cache@v2.1.4
         with:
-          path: ~/.cargo/registry
+          # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
           # Add date to the cache to keep it up to date
-          key: ${{ matrix.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
+          key: ${{ matrix.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
           # Restore from outdated cache for speed
           restore-keys: |
-            ${{ matrix.os }}-cargo-registry-${{ hashFiles('**/Cargo.toml') }}
+            ${{ matrix.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-
+            ${{ matrix.os }}-cargo-${{ github.job }}-
+            ${{ matrix.os }}-cargo-
 
-      - name: Cache cargo index
-        uses: actions/cache@v2.1.4
-        with:
-          path: ~/.cargo/git
-          # Add date to the cache to keep it up to date
-          key: ${{ matrix.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
-          # Restore from outdated cache for speed
-          restore-keys: |
-            ${{ matrix.os }}-cargo-index-${{ hashFiles('**/Cargo.toml') }}
-
-      - name: Cache cargo target
+      - name: Cache build target
         uses: actions/cache@v2.1.4
         with:
           path: target
           # Add date to the cache to keep it up to date
-          key: ${{ matrix.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
+          key: ${{ matrix.os }}-target-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
           # Restore from outdated cache for speed
           restore-keys: |
-            ${{ matrix.os }}-cargo-build-target-${{ hashFiles('**/Cargo.toml') }}
+            ${{ matrix.os }}-target-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-
+            ${{ matrix.os }}-target-${{ github.job }}-
+            ${{ matrix.os }}-target-
 
-      - name: Cache wasm cargo target
-        uses: actions/cache@v2.1.4
+      - name: Cache sccache
+        uses: actions/cache@v2.1.6
+        continue-on-error: false
         with:
-          path: bindings/wasm/target
-          # Add date to the cache to keep it up to date
-          key: ${{ matrix.os }}-cargo-build-wasm-target-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
-          # Restore from outdated cache for speed
+          path: ${{ matrix.sccache-path }}
+          key: ${{ runner.os }}-sccache-${{ github.job }}-${{ hashFiles('**/Cargo.lock') }}-${{ env.CURRENT_DATE }}
           restore-keys: |
-            ${{ matrix.os }}-cargo-build-wasm-target-${{ hashFiles('**/Cargo.toml') }}
+            ${{ runner.os }}-sccache-${{ github.job }}-${{ hashFiles('**/Cargo.lock') }}-
+            ${{ runner.os }}-sccache-${{ github.job }}-
+            ${{ runner.os }}-sccache-
 
-      - name: Cache libjose cargo target
-        uses: actions/cache@v2.1.4
-        with:
-          path: libjose/target
-          # Add date to the cache to keep it up to date
-          key: ${{ matrix.os }}-cargo-build-libjose-target-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
-          # Restore from outdated cache for speed
-          restore-keys: |
-            ${{ matrix.os }}-cargo-build-libjose-target-${{ hashFiles('**/Cargo.toml') }}
+      - name: Install sccache (ubuntu-latest)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          SCCACHE_PREFIX="sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl"
+          SCCACHE_TAR="${SCCACHE_PREFIX}.tar.gz"
+          DOWNLOAD_LINK="${SCCACHE_DOWNLOAD_LINK}/${SCCACHE_VERSION}/${SCCACHE_TAR}"
+          curl -L "${DOWNLOAD_LINK}" --output ${SCCACHE_TAR}
+          echo "$(curl -L ${DOWNLOAD_LINK}.sha256)  ${SCCACHE_TAR}" | shasum -a 256 --check --status
+          tar xzf ${SCCACHE_TAR}
+          BIN_DIR="$HOME/.local/bin"
+          mkdir -p ${BIN_DIR}
+          mv -f ${SCCACHE_PREFIX}/sccache ${BIN_DIR}/sccache
+          chmod a+x "${BIN_DIR}/sccache"
+          echo ${BIN_DIR} >> $GITHUB_PATH
+
+      - name: Install sccache (macos-latest)
+        if: matrix.os == 'macos-latest'
+        run: |
+          brew update
+          brew install sccache
+
+      - name: Install sccache (windows-latest)
+        if: matrix.os == 'windows-latest'
+        run: |
+          Invoke-Expression (New-Object System.Net.WebClient).DownloadString('https://get.scoop.sh')
+          scoop install sccache
+          echo "${HOME}/scoop/apps/sccache/current" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
+
+      - name: Start sccache
+        run: |
+          sccache --start-server
+          sccache -s
 
       - name: Build
         uses: actions-rs/cargo@v1
@@ -101,10 +142,23 @@ jobs:
           command: test
           args: --all --release
 
+      - name: Print sccache stats
+        run: sccache --show-stats
+
+      - name: Stop sccache server
+        run: sccache --stop-server || true
+
   build-and-test-libjose:
     runs-on: ubuntu-latest
     strategy:
-        fail-fast: false
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        include:
+          - os: ubuntu-latest
+            sccache-path: /home/runner/.cache/sccache
+    env:
+      SCCACHE_DIR: ${{ matrix.sccache-path }}
 
     steps:
       - uses: actions/checkout@v2
@@ -115,6 +169,69 @@ jobs:
           toolchain: stable
           override: true
 
+      - name: Get current date
+        run: echo "CURRENT_DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+      - name: Cache cargo
+        uses: actions/cache@v2.1.4
+        with:
+          # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          # Add date to the cache to keep it up to date
+          key: ${{ matrix.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
+          # Restore from outdated cache for speed
+          restore-keys: |
+            ${{ matrix.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-
+            ${{ matrix.os }}-cargo-${{ github.job }}-
+            ${{ matrix.os }}-cargo-
+
+      - name: Cache libjose build target
+        uses: actions/cache@v2.1.4
+        with:
+          path: libjose/target
+          # Add date to the cache to keep it up to date
+          key: ${{ matrix.os }}-libjose-target-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
+          # Restore from outdated cache for speed
+          restore-keys: |
+            ${{ matrix.os }}-libjose-target-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-
+            ${{ matrix.os }}-libjose-target-${{ github.job }}-
+            ${{ matrix.os }}-libjose-target-
+
+      - name: Cache sccache
+        uses: actions/cache@v2.1.6
+        continue-on-error: false
+        with:
+          path: ${{ matrix.sccache-path }}
+          key: ${{ runner.os }}-sccache-${{ github.job }}-${{ hashFiles('**/Cargo.lock') }}-${{ env.CURRENT_DATE }}
+          restore-keys: |
+            ${{ runner.os }}-sccache-${{ github.job }}-${{ hashFiles('**/Cargo.lock') }}-
+            ${{ runner.os }}-sccache-${{ github.job }}-
+            ${{ runner.os }}-sccache-
+
+      - name: Install sccache (ubuntu-latest)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          SCCACHE_PREFIX="sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl"
+          SCCACHE_TAR="${SCCACHE_PREFIX}.tar.gz"
+          DOWNLOAD_LINK="${SCCACHE_DOWNLOAD_LINK}/${SCCACHE_VERSION}/${SCCACHE_TAR}"
+          curl -L "${DOWNLOAD_LINK}" --output ${SCCACHE_TAR}
+          echo "$(curl -L ${DOWNLOAD_LINK}.sha256)  ${SCCACHE_TAR}" | shasum -a 256 --check --status
+          tar xzf ${SCCACHE_TAR}
+          BIN_DIR="$HOME/.local/bin"
+          mkdir -p ${BIN_DIR}
+          mv -f ${SCCACHE_PREFIX}/sccache ${BIN_DIR}/sccache
+          chmod a+x "${BIN_DIR}/sccache"
+          echo ${BIN_DIR} >> $GITHUB_PATH
+
+      - name: Start sccache
+        run: |
+          sccache --start-server
+          sccache -s
+
       - name: Build
         uses: actions-rs/cargo@v1
         with:
@@ -127,10 +244,23 @@ jobs:
           command: test
           args: --manifest-path ./libjose/Cargo.toml --release
 
+      - name: Print sccache stats
+        run: sccache --show-stats
+
+      - name: Stop sccache server
+        run: sccache --stop-server || true
+
   build-and-test-wasm:
     runs-on: ubuntu-latest
     strategy:
-        fail-fast: false
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        include:
+          - os: ubuntu-latest
+            sccache-path: /home/runner/.cache/sccache
+    env:
+      SCCACHE_DIR: ${{ matrix.sccache-path }}
 
     steps:
       - uses: actions/checkout@v2
@@ -147,6 +277,66 @@ jobs:
           toolchain: stable
           target: wasm32-unknown-unknown
 
+      - name: Cache cargo
+        uses: actions/cache@v2.1.4
+        with:
+          # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
+          path: |
+            ~/.cargo/bin/
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          # Add date to the cache to keep it up to date
+          key: ${{ matrix.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
+          # Restore from outdated cache for speed
+          restore-keys: |
+            ${{ matrix.os }}-cargo-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-
+            ${{ matrix.os }}-cargo-${{ github.job }}-
+            ${{ matrix.os }}-cargo-
+
+      - name: Cache wasm target
+        uses: actions/cache@v2.1.4
+        with:
+          path: bindings/wasm/target
+          # Add date to the cache to keep it up to date
+          key: ${{ matrix.os }}-wasm-target-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-${{ env.CURRENT_DATE }}
+          # Restore from outdated cache for speed
+          restore-keys: |
+            ${{ matrix.os }}-wasm-target-${{ github.job }}-${{ hashFiles('**/Cargo.toml') }}-
+            ${{ matrix.os }}-wasm-target-${{ github.job }}-
+            ${{ matrix.os }}-wasm-target-
+
+      - name: Cache sccache
+        uses: actions/cache@v2.1.6
+        continue-on-error: false
+        with:
+          path: ${{ matrix.sccache-path }}
+          key: ${{ runner.os }}-sccache-${{ github.job }}-${{ hashFiles('**/Cargo.lock') }}-${{ env.CURRENT_DATE }}
+          restore-keys: |
+            ${{ runner.os }}-sccache-${{ github.job }}-${{ hashFiles('**/Cargo.lock') }}-
+            ${{ runner.os }}-sccache-${{ github.job }}-
+            ${{ runner.os }}-sccache-
+
+      - name: Install sccache (ubuntu-latest)
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          SCCACHE_PREFIX="sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl"
+          SCCACHE_TAR="${SCCACHE_PREFIX}.tar.gz"
+          DOWNLOAD_LINK="${SCCACHE_DOWNLOAD_LINK}/${SCCACHE_VERSION}/${SCCACHE_TAR}"
+          curl -L "${DOWNLOAD_LINK}" --output ${SCCACHE_TAR}
+          echo "$(curl -L ${DOWNLOAD_LINK}.sha256)  ${SCCACHE_TAR}" | shasum -a 256 --check --status
+          tar xzf ${SCCACHE_TAR}
+          BIN_DIR="$HOME/.local/bin"
+          mkdir -p ${BIN_DIR}
+          mv -f ${SCCACHE_PREFIX}/sccache ${BIN_DIR}/sccache
+          chmod a+x "${BIN_DIR}/sccache"
+          echo ${BIN_DIR} >> $GITHUB_PATH
+
+      - name: Start sccache
+        run: |
+          sccache --start-server
+          sccache -s
+
       - name: Set up Node.js
         uses: actions/setup-node@v1
         with:
@@ -168,3 +358,9 @@ jobs:
         with:
           command: test
           args: --manifest-path ./bindings/wasm/Cargo.toml --release
+
+      - name: Print sccache stats
+        run: sccache --show-stats
+
+      - name: Stop sccache server
+        run: sccache --stop-server || true