ci: improve deps downloading and caching (#584)

* ci: improve deps downloading and caching

Bring down deps faster (brought over from work done on rewrite-clj).

Enable cross OS caching for GitHub Actions deps.
This should allow Windows runner to benefit from setup job caching.

Closes #577

* ci: don't convert line endings on clone on Windows

We want consistent hashes for our cross-os caches.
Our hashes are based on file contents.
We need the file contents to be the same across OSes.

* ci: experiment with fixing cross-os cache for win

Restoring the cache on Windows is awkward, I've raised an issue actions/cache#1400, I'll adapt based on response.

.github/workflows/release.yml

@@ -21,12 +21,13 @@ jobs:
       uses: fregante/setup-git-user@v2.0.1
 
     - name: Clojure deps cache
-      uses: actions/cache@v4
+      uses: actions/cache/restore@v4
       with:
         path: |
           ~/.m2/repository
           ~/.deps.clj
           ~/.gitlibs
+        enableCrossOsArchive: true
         key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }}
         restore-keys: cljdeps-
 

.github/workflows/test.yml

@@ -25,6 +25,7 @@ jobs:
           ~/.m2/repository
           ~/.deps.clj
           ~/.gitlibs
+        enableCrossOsArchive: true
         key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }}
         restore-keys: cljdeps-
 
@@ -59,6 +60,10 @@ jobs:
     name: ${{ matrix.desc }}
 
     steps:
+    - name: Don't convert line endings on checkout (Windows)
+      if: matrix.os == 'windows'
+      run: git config --global core.autocrlf false
+
     - name: Tune Windows network
       if: ${{ matrix.os == 'windows' }}
       run: Disable-NetAdapterChecksumOffload -Name * -TcpIPv4 -UdpIPv4 -TcpIPv6 -UdpIPv6
@@ -103,16 +108,28 @@ jobs:
 
     - uses: actions/checkout@v4
 
-    - name: Clojure deps cache
-      uses: actions/cache@v4
+    - name: Restore Clojure deps from cache
+      uses: actions/cache/restore@v4
       with:
         path: |
           ~/.m2/repository
           ~/.deps.clj
           ~/.gitlibs
+        enableCrossOsArchive: true
         key: cljdeps-${{ hashFiles('deps.edn', 'bb.edn') }}
         restore-keys: cljdeps-
 
+    - name: Cache fixup for Windows
+      # cache is restored using relative paths from Linux, this is not appropriate for Windows
+      # this is admitedly fragile/hackey, will revisit if it cracks
+      if: ${{ matrix.os == 'windows' }}
+      run: |
+        mv ../../../.m2 ${USERPROFILE}
+        mv ../../../.deps.clj ${USERPROFILE}
+        mv ../../../.gitlibs ${USERPROFILE}
+      shell: bash
+
+
     - name: Setup Java
       uses: actions/setup-java@v4
       with:

bb.edn

@@ -89,7 +89,7 @@
   tools-versions {:doc "report on tools versions"
                   :task tools-versions/-main}
   download-deps  {:doc "download all deps (useful for CI prep)"
-                  :task download-deps/-main}
+                  :task (clojure "-T:build download-deps")}
   outdated       {:doc "report on outdated dependencies"
                   :task (shell/clojure {:continue true} "-M:outdated")}
 

build.clj

@@ -1,8 +1,7 @@
 (ns build
   (:require [clojure.edn :as edn]
             [clojure.java.io :as io]
-            [clojure.tools.build.api :as b]
-            [deps-deploy.deps-deploy :as dd]))
+            [clojure.tools.build.api :as b]))
 
 (defn version-string []
   (let [{:keys [major minor release qualifier]} (-> "version.edn"
@@ -69,8 +68,24 @@
                 :jar-file jar-file})))
 
 (defn deploy [opts]
-  (dd/deploy
+  ((requiring-resolve 'deps-deploy.deps-deploy/deploy)
    {:installer :remote
     :artifact jar-file
     :pom-file (b/pom-path {:lib lib :class-dir class-dir})})
   opts)
+
+(defn download-deps
+  "Download all deps for all aliases"
+  [_]
+  (let [aliases (->> "deps.edn"
+                     slurp
+                     edn/read-string
+                     :aliases
+                     keys
+                     sort)]
+    ;; one at a time because aliases with :replace-deps will... well... you know.
+    (println "Bring down default deps")
+    (b/create-basis {})
+    (doseq [a (sort aliases)]
+      (println "Bring down deps for alias" a)
+      (b/create-basis {:aliases [a]}))))

deps.edn

@@ -55,10 +55,13 @@
                                                :add-linters [:performance]
                                                :exclude-linters [:local-shadows-var]}]}
 
-  :build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.0"}
-                 slipset/deps-deploy {:mvn/version "0.2.2"}}
+  :build {:deps {io.github.clojure/tools.build {:mvn/version "0.10.0"}}
           :ns-default build}
 
+  ;; keep deploy deps separate from build deps to avoid download-deps issues
+  ;; caused by, I think, conflicting maven deps
+  :deploy {:extra-deps {slipset/deps-deploy {:mvn/version "0.2.2"}}}
+
   :outdated {:extra-deps {com.github.liquidz/antq {:mvn/version  "2.8.1185"}
                           org.clojure/clojure {:mvn/version "1.11.1"}
                           org.slf4j/slf4j-simple {:mvn/version "2.0.12"} ;; to rid ourselves of logger warnings

script/ci_release.clj

@@ -126,7 +126,7 @@
   []
   (status/line :head "Deploying jar to clojars")
   (assert-on-ci "deploy a jar")
-  (shell/clojure "-T:build deploy")
+  (shell/clojure "-T:build:deploy deploy")
   nil)
 
 (defn- commit-changes! [version]