Improve ci/cd cache setup (#638)

* Only store build cache on master branch * Add comments on why the action looks how it looks * Add dev workflow * Always print test output * Update action * Use official actions for uploading and downloading artifacts * Run once a week * Use our own upload and download actions so we have sensible defaults for paths * Revert "Use our own upload and download actions so we have sensible defaults for paths" This reverts commit 4d9a063. * Fix missing paths * Fix defined versions
utopia-rise · May 20, 2024 · 66adfe1 · 66adfe1
1 parent 62e6edc
commit 66adfe1
Show file tree

Hide file tree

Showing 18 changed files with 312 additions and 54 deletions.
diff --git a/.github/actions/scons-cache/action.yml b/.github/actions/scons-cache/action.yml
@@ -0,0 +1,65 @@
+# base key setup copied from the godot cache action setup
+# just added some if statements for only restoring the cache when not on master and only saving when on master
+
+name: Setup scons cache
+description: Setup scons build cache.
+inputs:
+  cache-name:
+    description: The cache base name (job name by default).
+    default: "${{github.job}}"
+  scons-cache:
+    description: The scons cache path.
+    default: "${{github.workspace}}/.scons-cache/"
+
+runs:
+  using: "composite"
+  steps:
+    # Upload cache on completion and check it out now
+    - name: Load .scons_cache directory
+      if: ${{ github.ref == 'refs/heads/master' }}
+      uses: actions/cache@v4
+      with:
+        path: ${{inputs.scons-cache}}
+        key: ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+
+        # We try to match an existing cache to restore from it. Each potential key is checked against
+        # all existing caches as a prefix. E.g. 'linux-template-minimal' would match any cache that
+        # starts with "linux-template-minimal", such as "linux-template-minimal-master-refs/heads/master-6588a4a29af1621086feac0117d5d4d37af957fd".
+        #
+        # We check these prefixes in this order:
+        #
+        #   1. The exact match, including the base branch, the commit reference, and the SHA hash of the commit.
+        #   2. A partial match for the same base branch and the same commit reference.
+        #   3. A partial match for the same base branch and the base branch commit reference.
+        #   4. A partial match for the same base branch only (not ideal, matches any PR with the same base branch).
+
+        restore-keys: |
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-refs/heads/${{env.GODOT_BASE_BRANCH}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}
+
+    # Check out cache but do not upload it at the end
+    - name: Load .scons_cache directory
+      if: ${{ github.ref != 'refs/heads/master' }}
+      uses: actions/cache/restore@v4 # only restores the cache but does not upload it
+      with:
+        path: ${{inputs.scons-cache}}
+        key: ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+
+        # We try to match an existing cache to restore from it. Each potential key is checked against
+        # all existing caches as a prefix. E.g. 'linux-template-minimal' would match any cache that
+        # starts with "linux-template-minimal", such as "linux-template-minimal-master-refs/heads/master-6588a4a29af1621086feac0117d5d4d37af957fd".
+        #
+        # We check these prefixes in this order:
+        #
+        #   1. The exact match, including the base branch, the commit reference, and the SHA hash of the commit.
+        #   2. A partial match for the same base branch and the same commit reference.
+        #   3. A partial match for the same base branch and the base branch commit reference.
+        #   4. A partial match for the same base branch only (not ideal, matches any PR with the same base branch).
+
+        restore-keys: |
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}-${{github.sha}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-${{github.ref}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}-refs/heads/${{env.GODOT_BASE_BRANCH}}
+          ${{inputs.cache-name}}-${{env.GODOT_BASE_BRANCH}}
diff --git a/.github/workflows/assemble_export_templates.yml b/.github/workflows/assemble_export_templates.yml
@@ -37,6 +37,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_all_ios
+          path: "./"
 
       - name: Move ios export template
         shell: sh
@@ -46,6 +47,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_debug_linux_x86_64
+          path: "./"
 
       - name: Move linux x86_64 debug export template
         shell: sh
@@ -55,6 +57,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_release_linux_x86_64
+          path: "./"
 
       - name: Move linux x86_64 release export template
         shell: sh
@@ -64,6 +67,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_debug_windows_x86_64
+          path: "./"
 
       - name: Move windows x86_64 debug export template
         shell: sh
@@ -73,6 +77,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_release_windows_x86_64
+          path: "./"
 
       - name: Move windows x86_64 release export template
         shell: sh
@@ -82,6 +87,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_all_macos
+          path: "./"
 
       - name: Move macos export template
         shell: sh

diff --git a/.github/workflows/assemble_ios.yml b/.github/workflows/assemble_ios.yml
@@ -29,6 +29,7 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_ios_all
+          path: "./"
 
       - name: Clone MoltenVK
         uses: actions/checkout@v4

diff --git a/.github/workflows/assemble_macos.yml b/.github/workflows/assemble_macos.yml
@@ -38,11 +38,13 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: editor_${{ matrix.target }}_macos_x86_64
+          path: "./"
 
       - name: Download ${{ matrix.target }} arm64 editor
         uses: actions/download-artifact@v4
         with:
           name: editor_${{ matrix.target }}_macos_arm64
+          path: "./"
 
       - name: Create ${{ matrix.target }} OSX universal binary
         uses: ./.github/actions/create-macos-universal-binary
@@ -129,11 +131,13 @@ jobs:
         uses: actions/download-artifact@v4
         with:
           name: export_template_${{ matrix.target }}_macos_x86_64
+          path: "./"
 
       - name: Download ${{ matrix.target }} arm64 export_template
         uses: actions/download-artifact@v4
         with:
           name: export_template_${{ matrix.target }}_macos_arm64
+          path: "./"
 
       - name: Create macos universal binary
         uses: ./.github/actions/create-macos-universal-binary

diff --git a/.github/workflows/build_android.yml b/.github/workflows/build_android.yml
@@ -36,10 +36,10 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
         with:
-          cache-read-only: false
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
 
       - name: Setup Godot build cache
-        uses: ./.github/actions/godot-cache
+        uses: ./.github/actions/scons-cache
         continue-on-error: true
 
       - name: Setup python and scons
@@ -95,6 +95,7 @@ jobs:
           ls -l bin/
 
       - name: Upload artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: export_template_android_all
+          path: "bin/*"
diff --git a/.github/workflows/build_ios.yml b/.github/workflows/build_ios.yml
@@ -27,7 +27,7 @@ jobs:
           path: modules/kotlin_jvm
 
       - name: Setup Godot build cache
-        uses: ./.github/actions/godot-cache
+        uses: ./.github/actions/scons-cache
         continue-on-error: true
 
       - name: Setup python and scons
@@ -48,6 +48,7 @@ jobs:
           target: template_debug
 
       - name: Upload artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: export_template_ios_all
+          path: "bin/*"
diff --git a/.github/workflows/build_jvm.yml b/.github/workflows/build_jvm.yml
@@ -36,57 +36,57 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
         with:
-          cache-read-only: false
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
 
       - name: Compilation
         # we build the release bootstrap jar here. Debug gets its own trigger later
         run: |
           modules/kotlin_jvm/kt/gradlew -p modules/kotlin_jvm/kt/ build -Prelease
 
       - name: Upload api-generator artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_api-generator
           path: modules/kotlin_jvm/kt/api-generator/build/libs/api-generator.jar
 
       - name: Upload release bootstrap artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_godot-bootstrap_release
           path: modules/kotlin_jvm/kt/godot-library/build/libs/godot-bootstrap.jar
 
       - name: Upload entry-generator artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_entry-generator
           path: modules/kotlin_jvm/kt/entry-generation/godot-entry-generator/build/libs/godot-entry-generator-*.jar
 
       - name: Upload ksp-symbol-processor artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_ksp-symbol-processor
           path: modules/kotlin_jvm/kt/entry-generation/godot-kotlin-symbol-processor/build/libs/godot-kotlin-symbol-processor-*.jar
 
       - name: Upload gradle-plugin artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_gradle-plugin
           path: modules/kotlin_jvm/kt/plugins/godot-gradle-plugin/build/libs/godot-gradle-plugin-*.jar
 
       - name: Upload intellij-plugin artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_intellij-plugin
           path: modules/kotlin_jvm/kt/plugins/godot-intellij-plugin/build/distributions/godot-jvm-idea-plugin-*.zip
 
       - name: Upload plugins-common artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_plugins-common
           path: modules/kotlin_jvm/kt/plugins/godot-plugins-common/build/libs/godot-plugins-common-*.jar
 
       - name: Upload tools-common artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_tools-common
           path: modules/kotlin_jvm/kt/tools-common/build/libs/tools-common-*.jar
@@ -96,7 +96,7 @@ jobs:
           modules/kotlin_jvm/kt/gradlew -p modules/kotlin_jvm/kt/ build -Pdebug
 
       - name: Upload debug bootstrap artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: jvm_godot-bootstrap_debug
           path: modules/kotlin_jvm/kt/godot-library/build/libs/godot-bootstrap.jar
diff --git a/.github/workflows/build_linux.yml b/.github/workflows/build_linux.yml
@@ -55,7 +55,7 @@ jobs:
           path: modules/kotlin_jvm
 
       - name: Setup Godot build cache
-        uses: ./.github/actions/godot-cache
+        uses: ./.github/actions/scons-cache
         with:
           cache-name: ${{ matrix.cache-name }}
         continue-on-error: true
@@ -77,6 +77,7 @@ jobs:
           chmod +x bin/godot.*
 
       - name: Upload artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.cache-name }}
+          path: "bin/*"
diff --git a/.github/workflows/build_macos.yml b/.github/workflows/build_macos.yml
@@ -80,7 +80,7 @@ jobs:
           path: modules/kotlin_jvm
 
       - name: Setup Godot build cache
-        uses: ./.github/actions/godot-cache
+        uses: ./.github/actions/scons-cache
         with:
           cache-name: ${{ matrix.cache-name }}
         continue-on-error: true
@@ -105,6 +105,7 @@ jobs:
           chmod +x bin/godot.*
 
       - name: Upload artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.cache-name }}
+          path: "bin/*"
diff --git a/.github/workflows/build_windows.yml b/.github/workflows/build_windows.yml
@@ -64,7 +64,7 @@ jobs:
           path: modules/kotlin_jvm
 
       - name: Setup Godot build cache
-        uses: ./.github/actions/godot-cache
+        uses: ./.github/actions/scons-cache
         with:
           cache-name: ${{ matrix.cache-name }}
         continue-on-error: true
@@ -92,6 +92,7 @@ jobs:
           Remove-Item bin/* -Include *.pdb -Force
 
       - name: Upload artifact
-        uses: ./.github/actions/upload-artifact
+        uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.cache-name }}
+          path: "bin/*"
diff --git a/.github/workflows/deploy_create_draft_release.yml b/.github/workflows/deploy_create_draft_release.yml
@@ -21,7 +21,7 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
         with:
-          cache-read-only: false
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
 
       - name: Generate changelog
         shell: sh

diff --git a/.github/workflows/deploy_jvm.yml b/.github/workflows/deploy_jvm.yml
@@ -45,7 +45,7 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
         with:
-          cache-read-only: false
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
 
       - name: Publish tools-common
         shell: sh
@@ -112,7 +112,7 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
         with:
-          cache-read-only: false
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
 
       - name: Deploy intellij plugin ${{ matrix.ij_sdk }}
         shell: sh

diff --git a/.github/workflows/test_linux.yml b/.github/workflows/test_linux.yml
@@ -45,16 +45,16 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
         with:
-          cache-read-only: false
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
 
       - name: Download linux editor ${{ matrix.target }}
-        uses: ./.github/actions/download-artifact
+        uses: actions/download-artifact@v4
         with:
           name: editor_${{ matrix.target }}_linux_x86_64
           path: './bin'
 
       - name: Download godot-bootstrap ${{ matrix.bootstrap-target }}
-        uses: ./.github/actions/download-artifact
+        uses: actions/download-artifact@v4
         with:
           name: jvm_godot-bootstrap_${{ matrix.bootstrap-target }}
           path: './bin'

diff --git a/.github/workflows/test_macos.yml b/.github/workflows/test_macos.yml
@@ -45,16 +45,16 @@ jobs:
       - name: Setup Gradle
         uses: gradle/actions/setup-gradle@v3
         with:
-          cache-read-only: false
+          cache-read-only: ${{ github.ref != 'refs/heads/master' }}
 
       - name: Download macos editor ${{ matrix.target }}
-        uses: ./.github/actions/download-artifact
+        uses: actions/download-artifact@v4
         with:
           name: editor_${{ matrix.target }}_macos_universal
           path: './bin'
 
       - name: Download godot-bootstrap ${{ matrix.bootstrap-target }}
-        uses: ./.github/actions/download-artifact
+        uses: actions/download-artifact@v4
         with:
           name: jvm_godot-bootstrap_${{ matrix.bootstrap-target }}
           path: './bin'