Bring hotload-dependency back

CHANGELOG.md

@@ -17,6 +17,7 @@
 * Build ASTs more robustly (by using locks, `require`, and ruling out certain namespaces like refactor-nrepl itself)
 * Improve `namespace-aliases` performance and make it return more accurate results.
 * Honor internal `future-cancel` calls, improving overall responsiveness and stability.
+* [clojure-emacs/clj-refactor.el#466](https://github.com/clojure-emacs/clj-refactor.el/issues/466): Bring the `hotload-dependency` operation back.
 
 ### Bugs fixed
 

project.clj

@@ -5,6 +5,16 @@
             :url "http://www.eclipse.org/legal/epl-v10.html"}
   :dependencies [[nrepl "0.8.3"]
                  ^:inline-dep [http-kit "2.5.3"]
+                 ^:inline-dep [org.clojure/tools.deps.alpha "0.12.1048"
+                               :exclusions
+                               [
+                                ;; `javax.inject` is only needed for mvn repos
+                                ;;  in S3 and that lib doesn't work with
+                                ;;  mranderson at present, so we might as well
+                                ;;  exlude the related s3 libs
+                                com.cognitect.aws/s3
+                                com.cognitect.aws/endpoints
+                                javax.inject]]
                  ^:inline-dep [org.clojure/data.json "2.3.1"]
                  ^:inline-dep [org.clojure/tools.analyzer.jvm "1.1.0"]
                  ^:inline-dep [org.clojure/tools.namespace "1.1.0" :exclusions [org.clojure/tools.reader]]

src/refactor_nrepl/add_lib.clj

@@ -0,0 +1,91 @@
+;;   Copyright (c) Rich Hickey. All rights reserved.
+;;   The use and distribution terms for this software are covered by the
+;;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+;;   which can be found in the file epl-v10.html at the root of this distribution.
+;;   By using this software in any fashion, you are agreeing to be bound by
+;;   the terms of this license.
+;;   You must not remove this notice, or any other, from this software.
+
+(ns refactor-nrepl.add-lib
+  (:require
+   [clojure.java.io :as jio]
+   [clojure.set :as set]
+   [clojure.tools.deps.alpha :as deps]
+   [clojure.tools.deps.alpha.util.maven :as maven])
+  (:import
+   clojure.lang.DynamicClassLoader
+   java.io.File))
+
+(set! *warn-on-reflection* true)
+
+;; maintain basis
+
+(defn- read-basis
+  []
+  (when-let [f (jio/file (System/getProperty "clojure.basis"))]
+    (if (and f (.exists f))
+      (deps/slurp-deps f)
+      (throw (IllegalArgumentException. "No basis declared in clojure.basis system property")))))
+
+(defonce ^:private init-basis (delay (read-basis)))
+
+(defn launch-basis
+  "Initial runtime basis at launch"
+  []
+  @init-basis)
+
+(def ^:private runtime-basis
+  (atom nil))
+
+(defn- reset-basis
+  [basis]
+  (reset! runtime-basis basis))
+
+(defn current-basis
+  "Return the current runtime basis, which may have been modified since the launch."
+  []
+  (or @runtime-basis (reset-basis @init-basis)))
+
+;; add-libs
+
+(defn- add-loader-url
+  "Add url string or URL to the highest level DynamicClassLoader url set."
+  [url]
+  (let [u (if (string? url) (java.net.URL. url) url)
+        loader (loop [loader (.getContextClassLoader (Thread/currentThread))]
+                 (let [parent (.getParent loader)]
+                   (if (instance? DynamicClassLoader parent)
+                     (recur parent)
+                     loader)))]
+    (if (instance? DynamicClassLoader loader)
+      (.addURL ^DynamicClassLoader loader u)
+      (throw (IllegalAccessError. "Context classloader is not a DynamicClassLoader")))))
+
+(defn add-libs
+  "Add map of lib to coords to the current runtime environment. All transitive
+  dependencies will also be considered (in the context of the current set
+  of loaded dependencies) and new transitive dependencies will also be
+  loaded. Returns seq of all added libs or nil if couldn't be loaded.
+  Note that for successful use, you must be in a REPL environment where a
+  valid parent DynamicClassLoader can be found in which to add the new lib
+  urls.
+  Example:
+   (add-libs '{org.clojure/core.memoize {:mvn/version \"0.7.1\"}})"
+  [lib-coords]
+  (let [{:keys [libs] :as initial-basis} (current-basis)]
+    (if (empty? (set/difference (-> lib-coords keys set) (-> libs keys set)))
+      nil ;; already loaded
+      (let [updated-deps (reduce-kv (fn [m k v] (assoc m k (dissoc v :dependents :paths))) lib-coords libs)
+            updated-edn (merge (dissoc initial-basis :libs :classpath :deps) {:deps updated-deps})
+            {updated-libs :libs :as updated-basis}
+            (deps/calc-basis
+             ;; No `:mvn/repos` are configured if Leiningen is in use so we have to add them here.
+             (merge {:mvn/repos maven/standard-repos} updated-edn)
+             (select-keys initial-basis [:resolve-args :cp-args]))
+            new-libs (select-keys updated-libs (set/difference (set (keys updated-libs)) (set (keys libs))))
+            paths (mapcat :paths (vals new-libs))
+            urls (->> paths (map jio/file) (map #(.toURL ^File %)))]
+        ;; TODO: multiple unsynchronized changes to runtime state - coordinate with lock?
+        (run! add-loader-url urls)
+        (reset-basis updated-basis)
+        (keys new-libs)))))

src/refactor_nrepl/artifacts.clj

@@ -1,12 +1,20 @@
 (ns refactor-nrepl.artifacts
-  (:require [clojure.data.json :as json]
-            [clojure
-             [edn :as edn]
-             [string :as str]]
-            [clojure.java.io :as io]
-            [org.httpkit.client :as http]
-            [version-clj.core :as versions])
-  (:import java.util.zip.GZIPInputStream))
+  (:require
+   [clojure.data.json :as json]
+   [clojure.edn :as edn]
+   [clojure.java.io :as io]
+   [clojure.string :as str]
+   [refactor-nrepl.add-lib :as add-lib]
+   [clojure.tools.namespace.find :as find]
+   [org.httpkit.client :as http]
+   [refactor-nrepl.core :as core]
+   [refactor-nrepl.ns.slam.hound.regrow :as slamhound-regrow]
+   [refactor-nrepl.ns.slam.hound.search :as slamhound]
+   [version-clj.core :as versions])
+  (:import
+   java.io.File
+   java.util.jar.JarFile
+   java.util.zip.GZIPInputStream))
 
 (def artifacts-file (str (io/file (System/getProperty "java.io.tmpdir")
                                   "refactor-nrepl-artifacts-cache")))
@@ -17,13 +25,15 @@
   (let [lm (.lastModified (io/file file))]
     (if (zero? lm) nil lm)))
 
-;;  structure here is {"prismatic/schem" ["0.1.1" "0.2.0" ...]}
-(defonce artifacts (atom (if (.exists (io/file artifacts-file))
-                           (->> artifacts-file slurp edn/read-string (into {}))
-                           {})
-                         :meta {:last-modified
-                                (get-last-modified-from-file artifacts-file)}))
-
+;;  structure here is (mostly) {"prismatic/schem" ["0.1.1" "0.2.0" ...]}
+;;  The exceptions are the mvn based artifacts.  There's a ratelimit in place
+;;  for those artifacts so we get the available versions on demand instead.
+(defonce artifacts
+  (atom (if (.exists (io/as-file artifacts-file))
+          (->> artifacts-file slurp edn/read-string (into {}))
+          {})
+        :meta {:last-modified
+               (get-last-modified-from-file artifacts-file)}))
 (def millis-per-day (* 24 60 60 1000))
 
 (defn- get-proxy-opts
@@ -108,9 +118,7 @@
   (let [{:keys [body status]} @(http/get (str "https://clojars.org/api/artifacts/"
                                               artifact))]
     (when (= 200 status)
-      (->> (json/read-str body :key-fn keyword)
-           :recent_versions
-           (keep :version)))))
+      (map :version (:recent_versions (json/read-str body :key-fn keyword))))))
 
 (defn- get-artifacts-from-clojars!
   []
@@ -136,19 +144,84 @@
     (update-artifact-cache!))
   (->> @artifacts keys list*))
 
+(defn- artifact-versions* [artifact-id]
+  (->> (or (get @artifacts artifact-id)
+           (seq (get-mvn-versions! artifact-id))
+           (get-clojars-versions! artifact-id))
+       distinct
+       versions/version-sort
+       reverse
+       list*))
+
 (defn artifact-versions
   "Returns a sorted list of artifact version strings. The list can either come
   from the artifacts cache, the maven search api or the clojars search api in
   that order."
   [{:keys [artifact]}]
-  (->> (or (get @artifacts artifact)
-           (seq (get-mvn-versions! artifact))
-           (get-clojars-versions! artifact))
-       distinct
-       versions/version-sort
-       reverse
-       list*))
+  (artifact-versions* artifact))
+
+(defn- jar-at-the-top-of-dependency-hierarchy [top-level-dep]
+  ;; We only need to consider the dep at the top of the hierarchy because when we
+  ;; require those namespaces the rest of the transitive deps will get pulled in
+  ;; too.
+  (letfn [(->jar [^File f]
+            (JarFile. f))]
+    (let [artifact-name (-> top-level-dep core/suffix)]
+      (->> (core/jars-on-classpath)
+           ;; This isn't guaranteed but at least on my system the new stuff was
+           ;; added to the end and so this increases the likelihood of picking
+           ;; the right artifact
+           reverse
+           (map io/as-file)
+           (some (fn [f]
+                   (when (.startsWith (.getName f) artifact-name)
+                     f)))
+           ->jar))))
+
+(defn- make-resolve-missing-aware-of-new-deps!
+  "Once the deps are available on cp we still have to load them and
+  reset slamhound's cache to make resolve-missing work."
+  [^JarFile jar]
+  (doseq [new-namespace (find/find-namespaces-in-jarfile jar)]
+    (try
+      (require new-namespace)
+      (catch Exception _
+        ;; I've seen this happen after adding core.async as a dependency.
+        ;; It also happens if you try to require namespaces that no longer work,
+        ;; like compojure.handler.
+        ;; A failure here isn't a big deal, it only means that resolve-missing
+        ;; isn't going to work until the namespace has been loaded manually.
+        )))
+  (slamhound/reset)
+  (slamhound-regrow/clear-cache!))
+
+(defn- parse-coordinates [coordinates-str]
+  (let [coords (try (->> coordinates-str edn/read-string)
+                    (catch Exception _))]
+    (cond
+      ;; Leiningen dependency vector
+      (vector? coords)
+      (hash-map (first coords) {:mvn/version (second coords)})
+      ;; tools.deps map
+      (map? coords) coords
+      :else
+      (throw (IllegalArgumentException. (str "Malformed coordinates "
+                                             coordinates-str))))))
+
+(defn- add-dependencies! [coordinates]
+  ;; Just so we can mock this out during testing
+  (some-> coordinates
+          add-lib/add-libs
+          set
+          (apply (keys coordinates))))
+
+(defn- hotload-dependency! [coordinates]
+  (some-> coordinates
+          add-dependencies!
+          jar-at-the-top-of-dependency-hierarchy
+          make-resolve-missing-aware-of-new-deps!))
 
 (defn hotload-dependency
-  []
-  (throw (IllegalArgumentException. "Temporarily disabled until a solution for java 10 is found.")))
+  [{:keys [coordinates]}]
+  (when (->> coordinates parse-coordinates hotload-dependency!)
+    coordinates))

src/refactor_nrepl/core.clj

@@ -71,6 +71,10 @@
                        (-> s (.contains ".gitlibs"))))))
        (remove util/dir-outside-root-dir?)))
 
+(defn jars-on-classpath []
+  (->> (cp/classpath)
+       (filter misc/jar-file?)))
+
 (defn project-root
   "Return the project root directory.
 

test/refactor_nrepl/artifacts_test.clj

@@ -75,3 +75,17 @@
     (is (nil? (#'artifacts/edn-read-or-nil bad-form)))
     (is (= 'foo/bar (first (#'artifacts/edn-read-or-nil good-form))))
     (is (= "1.1" (second (#'artifacts/edn-read-or-nil good-form))))))
+
+(deftest hotload-dependency-throws-exceptions
+  (reset! artifacts/artifacts {"prismatic/schema" ["0.1"]})
+  (with-redefs
+    [artifacts/make-resolve-missing-aware-of-new-deps! (constantly true)
+     artifacts/stale-cache? (constantly false)
+     artifacts/jar-at-the-top-of-dependency-hierarchy (constantly true)
+     artifacts/add-dependencies! (constantly true)]
+    (testing "Throws for non existing version"
+      (is (thrown? IllegalArgumentException
+                   (artifacts/hotload-dependency
+                    {:coordinates "obviously wrong"}))))
+    (testing "No exception when all is OK"
+      (is (artifacts/hotload-dependency {:coordinates "[prismatic/schema \"0.1\"]"})))))