From da80d8f7c14bf7359e1572117ec3b84ff2ffc0bd Mon Sep 17 00:00:00 2001 From: Arne Brasseur Date: Fri, 16 Sep 2022 11:15:19 +0200 Subject: [PATCH] Implement .env support and hot-reloading Load environment variables from .env and .env.local, and also watch these files for changes. Java does not normally allow you to change the environment variables of the running JVM, and we go through some serious shenanigans to support it. We also unified the watching of deps.edn and .env so we don't have to start so many different beholder watchers. Other launchpad extensions can plug in to this mechanism as well to watch additional files. --- deps.edn | 3 +- src/lambdaisland/launchpad.clj | 97 +++++++++++++++++++------- src/lambdaisland/launchpad/deps.clj | 21 +++++- src/lambdaisland/launchpad/env.clj | 87 +++++++++++++++++++++++ src/lambdaisland/launchpad/watcher.clj | 61 ++++++++++++++++ 5 files changed, 241 insertions(+), 28 deletions(-) create mode 100644 src/lambdaisland/launchpad/env.clj create mode 100644 src/lambdaisland/launchpad/watcher.clj diff --git a/deps.edn b/deps.edn index 22fd43b..9acbc98 100644 --- a/deps.edn +++ b/deps.edn @@ -1,7 +1,8 @@ {:paths ["src" "resources"] :deps - {org.clojure/clojure {:mvn/version "1.11.1"}} + {org.clojure/clojure {:mvn/version "1.11.1"} + com.lambdaisland/dotenv {:mvn/version "0.1.1"}} :aliases {:dev diff --git a/src/lambdaisland/launchpad.clj b/src/lambdaisland/launchpad.clj index e434650..ae03e5b 100644 --- a/src/lambdaisland/launchpad.clj +++ b/src/lambdaisland/launchpad.clj @@ -8,7 +8,8 @@ [clojure.java.io :as io] [clojure.java.shell :as shell] [clojure.string :as str] - [clojure.tools.cli :as tools-cli]) + [clojure.tools.cli :as tools-cli] + [lambdaisland.dotenv :as dotenv]) (:import java.net.ServerSocket)) (def cli-opts @@ -22,9 +23,9 @@ (def default-nrepl-version "1.0.0") (def default-cider-version "0.28.3") (def default-refactor-nrepl-version "3.5.2") -(def classpath-coords - {:mvn/version "0.4.44"} - #_{:local/root "/home/arne/github/lambdaisland/classpath"}) + +(def classpath-coords {:mvn/version "0.4.44"}) +(def jnr-posix-coords {:mvn/version "3.1.15"}) (def default-launchpad-coords "Version coordinates for Launchpad, which we use to inject ourselves into the @@ -185,39 +186,76 @@ (Thread/sleep 3000) ctx) -(defn clojure-cli-args [{:keys [aliases nrepl-port middleware extra-deps eval-forms] :as ctx}] - (cond-> ["clojure" - "-J-XX:-OmitStackTraceInFastThrow" - (str "-J-Dlambdaisland.launchpad.aliases=" (str/join "," (map #(subs (str %) 1) aliases))) - #_(str "-J-Dlambdaisland.launchpad.extra-dep-source=" (pr-str {:deps extra-deps})) - ] +(defn disable-stack-trace-elision [ctx] + (update ctx + :java-args conj + "-XX:-OmitStackTraceInFastThrow")) + +(defn inject-aliases-as-property [{:keys [aliases] :as ctx}] + (update ctx + :java-args conj + (str "-Dlambdaisland.launchpad.aliases=" (str/join "," (map #(subs (str %) 1) aliases))))) + +(defn include-watcher [{:keys [watch-handlers] :as ctx}] + (if watch-handlers + (-> ctx + (update :requires conj 'lambdaisland.launchpad.watcher) + (update :eval-forms (fnil conj []) + `(lambdaisland.launchpad.watcher/watch! ~watch-handlers))))) + +(defn clojure-cli-args [{:keys [aliases requires nrepl-port java-args middleware extra-deps eval-forms] :as ctx}] + (cond-> ["clojure"] + :-> (into (map #(str "-J" %)) java-args) (seq aliases) (conj (str/join (cons "-A" aliases))) extra-deps (into ["-Sdeps" (pr-str {:deps extra-deps})]) :-> - (into ["-M" "-e" (pr-str `(do ~@eval-forms))]) + (into ["-M" "-e" (pr-str `(do ~(when (seq requires) + (list* 'require (map #(list 'quote %) requires))) + ~@eval-forms))]) middleware (into []))) (defn run-nrepl-server [{:keys [nrepl-port middleware] :as ctx}] - (update ctx :eval-forms (fnil conj []) - '(require 'nrepl.cmdline) - `(nrepl.cmdline/-main "--port" ~(str nrepl-port) "--middleware" ~(pr-str middleware)))) + (-> ctx + (update :requires conj 'nrepl.cmdline) + (update :eval-forms (fnil conj []) + `(nrepl.cmdline/-main "--port" ~(str nrepl-port) "--middleware" ~(pr-str middleware))))) + +(defn register-watch-handlers [ctx handlers] + (update ctx + :watch-handlers + (fn [h] + (if h + `(~'merge ~h ~handlers) + handlers)))) (defn include-hot-reload-deps [{:keys [extra-deps aliases] :as ctx}] (as-> ctx <> (update <> :extra-deps assoc 'com.lambdaisland/classpath classpath-coords) - (update <> :eval-forms (fnil conj []) - '(require 'lambdaisland.classpath.watch-deps - 'lambdaisland.launchpad.deps) - `(lambdaisland.classpath.watch-deps/start! - ~{:aliases (mapv keyword aliases) - :include-local-roots? true - :basis-fn 'lambdaisland.launchpad.deps/basis - :watch-paths (when (.exists (io/file "deps.local.edn")) - ['(lambdaisland.classpath.watch-deps/canonical-path "deps.local.edn")]) - :launchpad/extra-deps `'~(:extra-deps <>)})))) + (update <> :requires conj 'lambdaisland.launchpad.deps) + (register-watch-handlers + <> + `(lambdaisland.launchpad.deps/watch-handlers + ~{:aliases (mapv keyword aliases) + :include-local-roots? true + :basis-fn 'lambdaisland.launchpad.deps/basis + :watch-paths ['(lambdaisland.classpath.watch-deps/canonical-path "deps.local.edn")] + :launchpad/extra-deps `'~(:extra-deps <>)})))) + +(defn watch-dotenv [ctx] + (-> ctx + (update :extra-deps assoc 'com.github.jnr/jnr-posix jnr-posix-coords) + (update :java-args conj + "--add-opens=java.base/java.lang=ALL-UNNAMED" + "--add-opens=java.base/java.util=ALL-UNNAMED") + (update :env #(apply merge % (map (fn [p] + (when (.exists (io/file p)) + (dotenv/parse-dotenv (slurp p)))) + [".env" ".env.local"]))) + (update :requires conj 'lambdaisland.launchpad.env) + (register-watch-handlers '(lambdaisland.launchpad.env/watch-handlers)))) (defn start-shadow-build [{:keys [deps-edn aliases] :as ctx}] (let [build-ids (concat (:launchpad/shadow-build-ids deps-edn) @@ -279,14 +317,21 @@ read-deps-edn handle-cli-args compute-middleware + ;; inject dependencies and enable behavior compute-extra-deps include-hot-reload-deps include-launchpad-deps + watch-dotenv + ;; extra java flags + disable-stack-trace-elision + inject-aliases-as-property + ;; start the actual process + include-watcher run-nrepl-server start-process wait-for-nrepl - maybe-connect-emacs - ]) + ;; stuff that happens after the server is up + maybe-connect-emacs]) (defn find-project-root [] (loop [dir (.getParent (io/file *file*))] diff --git a/src/lambdaisland/launchpad/deps.clj b/src/lambdaisland/launchpad/deps.clj index c814cd0..9b9a3e4 100644 --- a/src/lambdaisland/launchpad/deps.clj +++ b/src/lambdaisland/launchpad/deps.clj @@ -2,7 +2,8 @@ (:require [clojure.edn :as edn] [clojure.java.io :as io] - [clojure.tools.deps.alpha :as deps])) + [clojure.tools.deps.alpha :as deps] + [lambdaisland.classpath.watch-deps :as watch-deps])) (defn basis [opts] (let [deps-local-file (io/file "deps.local.edn") @@ -12,3 +13,21 @@ (deps/create-basis (-> opts (update :aliases concat (:launchpad/aliases deps-local)) (assoc :extra extra-deps))))) + +(defn watch-handlers [opts] + (let [basis (basis opts) + deps-paths (cond-> [(watch-deps/path watch-deps/process-root-path "deps.edn")] + (:include-local-roots? opts) + (into (->> (vals (:libs basis)) + (keep :local/root) + (map watch-deps/canonical-path) + (map #(watch-deps/path % "deps.edn")))) + (string? (:extra opts)) + (conj (watch-deps/canonical-path (:extra opts))) + :always + (concat (:watch-paths opts))) + handler (partial #'watch-deps/on-event deps-paths opts)] + (into {} + (map (fn [p] + [(str p) handler])) + deps-paths))) diff --git a/src/lambdaisland/launchpad/env.clj b/src/lambdaisland/launchpad/env.clj new file mode 100644 index 0000000..a992a73 --- /dev/null +++ b/src/lambdaisland/launchpad/env.clj @@ -0,0 +1,87 @@ +(ns lambdaisland.launchpad.env + "Make environment variables modifiable from within Java, and use that to watch a + .env file for changes, and hot reload them. + + This is *very* dirty, it uses reflection to get at various private bits of + Java, it relies on implementation details of OpenJDK, and it requires breaking + module isolation (the process has to start with + `--add-opens=java.base/java.lang=ALL-UNNAMED` + `--add-opens=java.base/java.util=ALL-UNNAMED`). We also rely on jnr-posix to + get to the underlying setenv system call, for good measure. + + But hey it works!" + (:require [lambdaisland.dotenv :as dotenv]) + (:import (java.nio.file Path Files LinkOption))) + +(set! *warn-on-reflection* true) + +(defn accessible-field ^java.lang.reflect.Field [^Class klz field] + (doto (.getDeclaredField klz field) + (.setAccessible true))) + +(defn get-static [field] + (let [klz (Class/forName (namespace field))] + (.get (accessible-field klz + (name field)) klz))) + +(defn get-field [^Object instance field] + (.get (accessible-field (.getClass instance) (str field)) instance)) + +(defn set-field! [klz field obj val] + (.set (accessible-field klz field) obj val)) + +(defn set-static! [klz field val] + (set-field! klz field klz val)) + +(def ^java.util.Map theEnvironment + (get-static 'java.lang.ProcessEnvironment/theEnvironment)) + +(def ^java.lang.ProcessEnvironment$StringEnvironment theUnmodifiableEnvironment + (get-field (get-static 'java.lang.ProcessEnvironment/theUnmodifiableEnvironment) 'm)) + +(def ^jnr.posix.POSIX posix (jnr.posix.POSIXFactory/getPOSIX)) + +(defn new-value [^String str] + (assert (= -1 (.indexOf str "\u0000"))) + (let [^java.lang.reflect.Constructor init + (first (.getDeclaredConstructors java.lang.ProcessEnvironment$Value))] + (.setAccessible init true) + (.newInstance init (into-array Object ["XXX" (.getBytes "XXX")])))) + +(defn new-variable [^String str] + (assert (and (= -1 (.indexOf str "=")) + (= -1 (.indexOf str "\u0000")))) + (let [^java.lang.reflect.Constructor init + (first (.getDeclaredConstructors java.lang.ProcessEnvironment$Value))] + (.setAccessible init true) + (.newInstance init (into-array Object ["XXX" (.getBytes "XXX")])))) + +(defn setenv + ([env] + (run! (fn [[k v]] (setenv k v)) env)) + ([^String var ^String val] + ;; This one is used by ProcessBuilder + (.put theEnvironment (new-value var) (new-variable val)) + ;; This one is used by System/getenv + (.put theUnmodifiableEnvironment var val) + ;; Also change the actual OS environment for the process + (.setenv posix var val 1))) + +(defn exists? + "Does the given path exist." + [path] + (Files/exists path (into-array LinkOption []))) + +(defn dotenv-watch-handler [paths] + (let [paths (map #(Path/of % (into-array String [])) paths)] + (fn [_] + (setenv + (apply merge + (map #(when (exists? %) + (dotenv/parse-dotenv (Files/readString %))) + paths)))))) + +(defn watch-handlers [] + (let [h (dotenv-watch-handler [".env" ".env.local"])] + {".env" h + ".env.local" h})) diff --git a/src/lambdaisland/launchpad/watcher.clj b/src/lambdaisland/launchpad/watcher.clj new file mode 100644 index 0000000..b002178 --- /dev/null +++ b/src/lambdaisland/launchpad/watcher.clj @@ -0,0 +1,61 @@ +(ns lambdaisland.launchpad.watcher + "Higher level wrapper around Beholder. + + Beholder watches directories, not files. We want to watch specific files in + specific directories. This can be done with [[watch!]], which will start the + minimum number of watchers to cover all directories, and will dispatch to the + right handler based on the changed file." + (:require [nextjournal.beholder :as beholder]) + (:import java.util.regex.Pattern + java.nio.file.LinkOption + java.nio.file.Files + java.nio.file.Paths + java.nio.file.Path)) + +(defonce watchers (atom nil)) + +(defn path ^Path [root & args] + (if (and (instance? Path root) (not (seq args))) + root + (Paths/get (str root) (into-array String args)))) + +(defn canonical-path [p] + (.toRealPath (path p) (into-array LinkOption []))) + +(defn parent-path [p] + (.getParent (path p))) + +(require 'clojure.pprint) + +(defn watch! + "Watch a number of files, takes a map from filename (string) to + handler (receives a map with `:type` and `:path`, as with Beholder)." + [file->handler] + (let [file->handler (update-keys file->handler canonical-path) + directories (distinct (map parent-path (keys file->handler))) + ;; in case of nested directories, only watch the top-most one + directories (remove (fn [d] + (some #(and (not= d %) + (.startsWith d %)) directories)) + directories)] + (swap! watchers + (fn [w] + (when w + (run! beholder/stop w)) + (doall + (for [dir directories] + (beholder/watch + (fn [{:keys [type path] :as event}] + (when-let [f (get file->handler path)] + (try + (f event) + (catch Exception e + (prn e))))) + (str dir)))))))) + +(comment + (watch! + {"/home/arne/Gaiwan/slack-widgets/deps.edn" prn + "/home/arne/Gaiwan/slack-widgets/.env" prn + "/home/arne/Gaiwan/slack-widgets/.envx" prn + "/home/arne/Gaiwan/slack-widgets/backend/deps.edn" prn}))