Implement .env support with hot-reloading

deps.edn

@@ -1,7 +1,8 @@
 {:paths ["src" "resources"]
 
  :deps
- {org.clojure/clojure {:mvn/version "1.11.1"}}
+ {org.clojure/clojure {:mvn/version "1.11.1"}
+  com.lambdaisland/dotenv {:mvn/version "0.1.1"}}
 
  :aliases
  {:dev

src/lambdaisland/launchpad.clj

@@ -8,7 +8,8 @@
             [clojure.java.io :as io]
             [clojure.java.shell :as shell]
             [clojure.string :as str]
-            [clojure.tools.cli :as tools-cli])
+            [clojure.tools.cli :as tools-cli]
+            [lambdaisland.dotenv :as dotenv])
   (:import java.net.ServerSocket))
 
 (def cli-opts
@@ -22,9 +23,9 @@
 (def default-nrepl-version "1.0.0")
 (def default-cider-version "0.28.3")
 (def default-refactor-nrepl-version "3.5.2")
-(def classpath-coords
-  {:mvn/version "0.4.44"}
-  #_{:local/root "/home/arne/github/lambdaisland/classpath"})
+
+(def classpath-coords {:mvn/version "0.4.44"})
+(def jnr-posix-coords {:mvn/version "3.1.15"})
 
 (def default-launchpad-coords
   "Version coordinates for Launchpad, which we use to inject ourselves into the
@@ -185,39 +186,76 @@
   (Thread/sleep 3000)
   ctx)
 
-(defn clojure-cli-args [{:keys [aliases nrepl-port middleware extra-deps eval-forms] :as ctx}]
-  (cond-> ["clojure"
-           "-J-XX:-OmitStackTraceInFastThrow"
-           (str "-J-Dlambdaisland.launchpad.aliases=" (str/join "," (map #(subs (str %) 1) aliases)))
-           #_(str "-J-Dlambdaisland.launchpad.extra-dep-source=" (pr-str {:deps extra-deps}))
-           ]
+(defn disable-stack-trace-elision [ctx]
+  (update ctx
+          :java-args conj
+          "-XX:-OmitStackTraceInFastThrow"))
+
+(defn inject-aliases-as-property [{:keys [aliases] :as ctx}]
+  (update ctx
+          :java-args conj
+          (str "-Dlambdaisland.launchpad.aliases=" (str/join "," (map #(subs (str %) 1) aliases)))))
+
+(defn include-watcher [{:keys [watch-handlers] :as ctx}]
+  (if watch-handlers
+    (-> ctx
+        (update :requires conj 'lambdaisland.launchpad.watcher)
+        (update :eval-forms (fnil conj [])
+                `(lambdaisland.launchpad.watcher/watch! ~watch-handlers)))))
+
+(defn clojure-cli-args [{:keys [aliases requires nrepl-port java-args middleware extra-deps eval-forms] :as ctx}]
+  (cond-> ["clojure"]
+    :-> (into (map #(str "-J" %)) java-args)
     (seq aliases)
     (conj (str/join (cons "-A" aliases)))
     extra-deps
     (into ["-Sdeps" (pr-str {:deps extra-deps})])
     :->
-    (into ["-M" "-e" (pr-str `(do ~@eval-forms))])
+    (into ["-M" "-e" (pr-str `(do ~(when (seq requires)
+                                     (list* 'require (map #(list 'quote %) requires)))
+                                  ~@eval-forms))])
     middleware
     (into [])))
 
 (defn run-nrepl-server [{:keys [nrepl-port middleware] :as ctx}]
-  (update ctx :eval-forms (fnil conj [])
-          '(require 'nrepl.cmdline)
-          `(nrepl.cmdline/-main "--port" ~(str nrepl-port) "--middleware" ~(pr-str middleware))))
+  (-> ctx
+      (update :requires conj 'nrepl.cmdline)
+      (update :eval-forms (fnil conj [])
+              `(nrepl.cmdline/-main "--port" ~(str nrepl-port) "--middleware" ~(pr-str middleware)))))
+
+(defn register-watch-handlers [ctx handlers]
+  (update ctx
+          :watch-handlers
+          (fn [h]
+            (if h
+              `(~'merge ~h ~handlers)
+              handlers))))
 
 (defn include-hot-reload-deps [{:keys [extra-deps aliases] :as ctx}]
   (as-> ctx <>
     (update <> :extra-deps assoc 'com.lambdaisland/classpath classpath-coords)
-    (update <> :eval-forms (fnil conj [])
-            '(require 'lambdaisland.classpath.watch-deps
-                      'lambdaisland.launchpad.deps)
-            `(lambdaisland.classpath.watch-deps/start!
-              ~{:aliases (mapv keyword aliases)
-                :include-local-roots? true
-                :basis-fn 'lambdaisland.launchpad.deps/basis
-                :watch-paths (when (.exists (io/file "deps.local.edn"))
-                               ['(lambdaisland.classpath.watch-deps/canonical-path "deps.local.edn")])
-                :launchpad/extra-deps `'~(:extra-deps <>)}))))
+    (update <> :requires conj 'lambdaisland.launchpad.deps)
+    (register-watch-handlers
+     <>
+     `(lambdaisland.launchpad.deps/watch-handlers
+       ~{:aliases (mapv keyword aliases)
+         :include-local-roots? true
+         :basis-fn 'lambdaisland.launchpad.deps/basis
+         :watch-paths ['(lambdaisland.classpath.watch-deps/canonical-path "deps.local.edn")]
+         :launchpad/extra-deps `'~(:extra-deps <>)}))))
+
+(defn watch-dotenv [ctx]
+  (-> ctx
+      (update :extra-deps assoc 'com.github.jnr/jnr-posix jnr-posix-coords)
+      (update :java-args conj
+              "--add-opens=java.base/java.lang=ALL-UNNAMED"
+              "--add-opens=java.base/java.util=ALL-UNNAMED")
+      (update :env #(apply merge % (map (fn [p]
+                                          (when (.exists (io/file p))
+                                            (dotenv/parse-dotenv (slurp p))))
+                                        [".env" ".env.local"])))
+      (update :requires conj 'lambdaisland.launchpad.env)
+      (register-watch-handlers '(lambdaisland.launchpad.env/watch-handlers))))
 
 (defn start-shadow-build [{:keys [deps-edn aliases] :as ctx}]
   (let [build-ids (concat (:launchpad/shadow-build-ids deps-edn)
@@ -279,14 +317,21 @@
                     read-deps-edn
                     handle-cli-args
                     compute-middleware
+                    ;; inject dependencies and enable behavior
                     compute-extra-deps
                     include-hot-reload-deps
                     include-launchpad-deps
+                    watch-dotenv
+                    ;; extra java flags
+                    disable-stack-trace-elision
+                    inject-aliases-as-property
+                    ;; start the actual process
+                    include-watcher
                     run-nrepl-server
                     start-process
                     wait-for-nrepl
-                    maybe-connect-emacs
-                    ])
+                    ;; stuff that happens after the server is up
+                    maybe-connect-emacs])
 
 (defn find-project-root []
   (loop [dir (.getParent (io/file *file*))]

src/lambdaisland/launchpad/deps.clj

@@ -2,7 +2,8 @@
   (:require
    [clojure.edn :as edn]
    [clojure.java.io :as io]
-   [clojure.tools.deps.alpha :as deps]))
+   [clojure.tools.deps.alpha :as deps]
+   [lambdaisland.classpath.watch-deps :as watch-deps]))
 
 (defn basis [opts]
   (let [deps-local-file (io/file "deps.local.edn")
@@ -12,3 +13,21 @@
     (deps/create-basis (-> opts
                            (update :aliases concat (:launchpad/aliases deps-local))
                            (assoc :extra extra-deps)))))
+
+(defn watch-handlers [opts]
+  (let [basis (basis opts)
+        deps-paths (cond-> [(watch-deps/path watch-deps/process-root-path "deps.edn")]
+                     (:include-local-roots? opts)
+                     (into (->> (vals (:libs basis))
+                                (keep :local/root)
+                                (map watch-deps/canonical-path)
+                                (map #(watch-deps/path % "deps.edn"))))
+                     (string? (:extra opts))
+                     (conj (watch-deps/canonical-path (:extra opts)))
+                     :always
+                     (concat (:watch-paths opts)))
+        handler (partial #'watch-deps/on-event deps-paths opts)]
+    (into {}
+          (map (fn [p]
+                 [(str p) handler]))
+          deps-paths)))

src/lambdaisland/launchpad/env.clj

@@ -0,0 +1,87 @@
+(ns lambdaisland.launchpad.env
+  "Make environment variables modifiable from within Java, and use that to watch a
+  .env file for changes, and hot reload them.
+
+  This is *very* dirty, it uses reflection to get at various private bits of
+  Java, it relies on implementation details of OpenJDK, and it requires breaking
+  module isolation (the process has to start with
+  `--add-opens=java.base/java.lang=ALL-UNNAMED`
+  `--add-opens=java.base/java.util=ALL-UNNAMED`). We also rely on jnr-posix to
+  get to the underlying setenv system call, for good measure.
+
+  But hey it works!"
+  (:require [lambdaisland.dotenv :as dotenv])
+  (:import (java.nio.file Path Files LinkOption)))
+
+(set! *warn-on-reflection* true)
+
+(defn accessible-field ^java.lang.reflect.Field [^Class klz field]
+  (doto (.getDeclaredField klz field)
+    (.setAccessible true)))
+
+(defn get-static [field]
+  (let [klz (Class/forName (namespace field))]
+    (.get (accessible-field klz
+                            (name field)) klz)))
+
+(defn get-field [^Object instance field]
+  (.get (accessible-field (.getClass instance) (str field)) instance))
+
+(defn set-field! [klz field obj val]
+  (.set (accessible-field klz field) obj val))
+
+(defn set-static! [klz field val]
+  (set-field! klz field klz val))
+
+(def ^java.util.Map theEnvironment
+  (get-static 'java.lang.ProcessEnvironment/theEnvironment))
+
+(def ^java.lang.ProcessEnvironment$StringEnvironment theUnmodifiableEnvironment
+  (get-field (get-static 'java.lang.ProcessEnvironment/theUnmodifiableEnvironment) 'm))
+
+(def ^jnr.posix.POSIX posix (jnr.posix.POSIXFactory/getPOSIX))
+
+(defn new-value [^String str]
+  (assert (= -1 (.indexOf str "\u0000")))
+  (let [^java.lang.reflect.Constructor init
+        (first (.getDeclaredConstructors java.lang.ProcessEnvironment$Value))]
+    (.setAccessible init true)
+    (.newInstance init (into-array Object ["XXX" (.getBytes "XXX")]))))
+
+(defn new-variable [^String str]
+  (assert (and (= -1 (.indexOf str "="))
+               (= -1 (.indexOf str "\u0000"))))
+  (let [^java.lang.reflect.Constructor init
+        (first (.getDeclaredConstructors java.lang.ProcessEnvironment$Value))]
+    (.setAccessible init true)
+    (.newInstance init (into-array Object ["XXX" (.getBytes "XXX")]))))
+
+(defn setenv
+  ([env]
+   (run! (fn [[k v]] (setenv k v)) env))
+  ([^String var ^String val]
+   ;; This one is used by ProcessBuilder
+   (.put theEnvironment (new-value var) (new-variable val))
+   ;; This one is used by System/getenv
+   (.put theUnmodifiableEnvironment var val)
+   ;; Also change the actual OS environment for the process
+   (.setenv posix var val 1)))
+
+(defn exists?
+  "Does the given path exist."
+  [path]
+  (Files/exists path (into-array LinkOption [])))
+
+(defn dotenv-watch-handler [paths]
+  (let [paths (map #(Path/of % (into-array String [])) paths)]
+    (fn [_]
+      (setenv
+       (apply merge
+              (map #(when (exists? %)
+                      (dotenv/parse-dotenv (Files/readString %)))
+                   paths))))))
+
+(defn watch-handlers []
+  (let [h (dotenv-watch-handler [".env" ".env.local"])]
+    {".env" h
+     ".env.local" h}))

src/lambdaisland/launchpad/watcher.clj

@@ -0,0 +1,61 @@
+(ns lambdaisland.launchpad.watcher
+  "Higher level wrapper around Beholder.
+
+  Beholder watches directories, not files. We want to watch specific files in
+  specific directories. This can be done with [[watch!]], which will start the
+  minimum number of watchers to cover all directories, and will dispatch to the
+  right handler based on the changed file."
+  (:require [nextjournal.beholder :as beholder])
+  (:import java.util.regex.Pattern
+           java.nio.file.LinkOption
+           java.nio.file.Files
+           java.nio.file.Paths
+           java.nio.file.Path))
+
+(defonce watchers (atom nil))
+
+(defn path ^Path [root & args]
+  (if (and (instance? Path root) (not (seq args)))
+    root
+    (Paths/get (str root) (into-array String args))))
+
+(defn canonical-path [p]
+  (.toRealPath (path p) (into-array LinkOption [])))
+
+(defn parent-path [p]
+  (.getParent (path p)))
+
+(require 'clojure.pprint)
+
+(defn watch!
+  "Watch a number of files, takes a map from filename (string) to
+  handler (receives a map with `:type` and `:path`, as with Beholder)."
+  [file->handler]
+  (let [file->handler (update-keys file->handler canonical-path)
+        directories (distinct (map parent-path (keys file->handler)))
+        ;; in case of nested directories, only watch the top-most one
+        directories (remove (fn [d]
+                              (some #(and (not= d %)
+                                          (.startsWith d %)) directories))
+                            directories)]
+    (swap! watchers
+           (fn [w]
+             (when w
+               (run! beholder/stop w))
+             (doall
+              (for [dir directories]
+                (beholder/watch
+                 (fn [{:keys [type path] :as event}]
+                   (when-let [f (get file->handler path)]
+                     (try
+                       (f event)
+                       (catch Exception e
+                         (prn e)))))
+                 (str dir))))))))
+
+(comment
+  (watch!
+   {"/home/arne/Gaiwan/slack-widgets/deps.edn" prn
+    "/home/arne/Gaiwan/slack-widgets/.env" prn
+    "/home/arne/Gaiwan/slack-widgets/.envx" prn
+    "/home/arne/Gaiwan/slack-widgets/backend/deps.edn" prn}))