From ce11cba5862289beceeb5ede5fc0694232c8083c Mon Sep 17 00:00:00 2001 From: Bryan Maass Date: Thu, 2 Nov 2023 11:20:51 -0600 Subject: [PATCH 1/2] bb dev tasks facelift --- bb.edn | 115 +++++++++++++++++-------------- bb/build_and_run_jar_locally.clj | 1 + bb/{bb => }/cli.clj | 10 +-- bb/{bb => }/dl_and_run.clj | 41 ++++++----- bb/{bb => }/meta.clj | 10 +-- bb/{bb => }/quick_test.clj | 0 bb/retry.clj | 17 +++++ bb/{bb => }/tasks.clj | 2 +- watch_ci.clj | 74 -------------------- 9 files changed, 119 insertions(+), 151 deletions(-) create mode 100644 bb/build_and_run_jar_locally.clj rename bb/{bb => }/cli.clj (96%) rename bb/{bb => }/dl_and_run.clj (88%) rename bb/{bb => }/meta.clj (90%) rename bb/{bb => }/quick_test.clj (100%) create mode 100644 bb/retry.clj rename bb/{bb => }/tasks.clj (97%) delete mode 100755 watch_ci.clj diff --git a/bb.edn b/bb.edn index 0d28963..7660b0d 100644 --- a/bb.edn +++ b/bb.edn @@ -1,9 +1,10 @@ {:min-bb-version "0.9.162" - :paths ["bb"] + :paths ["."] :deps {escherize/bask {:git/url "https://github.com/escherize/bask.git" :git/sha "81cc9af3021d7689cfbddf0518a1e828f785f006"} table/table {:mvn/version "0.5.0"}} - :tasks {:requires [[bask.colors :as c] + :tasks {:requires [[babashka.process :refer [shell]] + [bask.colors :as c] [bb.cli :as cli] [bb.tasks :as t] [clojure.string :as str]] @@ -28,64 +29,75 @@ {:id :port :msg "What port should we run it on?" :short "-p" :long "--port PORT" :prompt :number :default 3337} {:id :socket-repl :msg "What port shall we open a socket repl on?" :short "-s" :long "--socket-repl SOCKETPORT"}))))} - check-branch - {:requires [[bb.dl-and-run :as dl]] - :doc "Check the status of a branch in CI." - :examples [["bb check-branch --branch master" "Check master branch github checks"]] - :task - (do (dl/check-gh-token!) - (let [branch-info (cli/menu! (current-task) - {:id :branch - :msg "What branch should we check?" - :short "-b" - :long "--branch BRANCH" - :choices (delay (t/list-branches (t/mb-env))) - :choices-doc "list of branches" - :prompt :select} - {:id :watch? - :msg "Watch until success?" - :short "-w" - :long "--watch"})] - (if (:watch? branch-info) - (loop [] - (let [check (dl/checks-for-branch branch-info)] - (println (str "\n----------[ " - (.format (java.text.SimpleDateFormat. "hh:mm:ss a") (java.util.Date.)) - " ]")) - (doseq [[s c] check] - (println (dl/pretty s) s (str "(" c ") "))) - (when-not (= (keys check) [:success]) - (Thread/sleep 5000) - (recur)))) - (doseq [[s c] (dl/checks-for-branch branch-info)] - (println (dl/pretty s) s (str "(" c ") "))))))} - metabuild {:doc "Starts metabase locally in dev mode. Set FORCE_MB_DB_CONNECTION_URI to override connection string building" :requires [[bb.meta :as meta]] :examples [["FORCE_MB_DB_CONNECTION_URI=mysql://localhost:3308/metabase_test?user=root bb metabuild -d mysql" "Connect to MYSQL, running against run-mariadb-latest.sh"]] - :task (let [{:keys [app-db user-name password extensions] :as p} + :task (let [_ (println (c/red "Welcome to " (c/on-white (c/blue " MetaBuilder ")))) + {:keys [app-db user-name password extensions] :as p} (cli/menu! (current-task) - {:id :app-db :short "-d" :long "--database-type DB" :title "Which DB should we use for metabase's data?" :choices ["postgres" "h2" "mysql"] :required? true :prompt :select} - {:id :user-name :short "-u" :long "--username USER" :default (t/whoami)} - {:id :password :short "-p" :long "--pw PW" :default "password"} - {:id :extensions :short "-e" :long "--exensions EXT" :default ["dev" "ee" "ee-dev" "drivers" "drivers-dev" "cider/nrepl"] :prompt :multi} - {:id :db-name :short "-n" :long "--name DB_NAME" :default "metabase" :required? true :title "Name of the database to connect to." :prompt :text})] + {:id :app-db + :short "-d" + :long "--database-type DB" + :title "Pick Metabase's app-db" + :choices ["postgres" "h2" "mysql"] + :prompt :select} + {:id :user-name :short "-u" :long "--username USER" :default (t/whoami)} + {:id :password :short "-p" :long "--pw PW" :default "password"} + {:id :extensions :short "-e" :long "--extensions EXT" :default ["dev" "ee" "ee-dev" "drivers" "drivers-dev" "cider/nrepl"] :prompt :multi} + {:id :db-name + :short "-n" + :long "--name DB_NAME" + :default "metabase" + :title "Name of the database to connect to." + :prompt :text})] ;; (prn p) (meta/build app-db user-name password extensions))} - quick-test {:doc "Quickly run a test against a namespace." - :requires [[bb.quick-test :as qt]] - :task (qt/go! - (:test-namespaces (cli/menu! (current-task) - {:id :test-namespaces - :msg "What namespace(s) to test?" - :long "--ns NS" - :required? true - :prompt :multi - :choices (delay (qt/test-nss)) - :choices-doc "a list of clojure test namespaces"})))} + quick-test + {:doc "Quickly run a test against a namespace." + :requires [[bb.quick-test :as qt]] + :task (qt/go! + (:test-namespaces (cli/menu! (current-task) + {:id :test-namespaces + :msg "What namespace(s) to test?" + :long "--ns NS" + :prompt :multi + :choices (delay (qt/test-nss)) + :choices-doc "a list of clojure test namespaces"})))} + + watch-ci + {:requires [[bb.dl-and-run :as dl] + [clojure.pprint :as pp]] + :doc "Watch the CI status for a branch until it passes." + :examples [["bb watch-ci --branch current" "Check currently checked out branch in MB_DIR repo"]] + :task + (do (dl/check-gh-token!) + (let [{:keys [branch]} (cli/menu! (current-task) + {:id :branch + :msg "What branch should we check?" + :short "-b" + :long "--branch BRANCH" + :choices (delay (t/list-branches (t/mb-env))) + :choices-doc "list of branches" + :prompt :select}) + branch (if (= "current" branch) + (let [current (str/trim + (:out (shell {:dir (t/env "MB_DIR" (fn [] + (println "Please set MB_DIR to your metabase repo!") + (System/exit 1))) :out :string} + "git rev-parse --abbrev-ref HEAD")))] + (do (println (c/green (c/bold "Checking current branch: " current))) + current)) + branch)] + (pp/pprint dl/pretty) + (loop [] + (let [check (dl/checks-for-branch branch)] + (print (str "\n[ " (.format (java.text.SimpleDateFormat. "hh:mm:ss a") (java.util.Date.)) " ]")) + (doseq [[status count] check] (print (str/join (repeat count (dl/pretty status)))) (print "|")) (flush) + (when-not (= (keys check) [:success]) + (Thread/sleep 10000) (recur))))))} install-autotab {:doc "Prints shell code to autocomplete tasks using bb. Note: for fish shell please make sure ~/.config/fish/completions exists." @@ -96,7 +108,6 @@ (cli/menu! (current-task) {:id :shell-type :long "--shell SHELL" - :required? true :title "What kind of shell?" :choices ["zsh" "bash" "fish"] :prompt :select})] diff --git a/bb/build_and_run_jar_locally.clj b/bb/build_and_run_jar_locally.clj new file mode 100644 index 0000000..0b3b781 --- /dev/null +++ b/bb/build_and_run_jar_locally.clj @@ -0,0 +1 @@ +(ns bb.build-and-run-jar-locally) diff --git a/bb/bb/cli.clj b/bb/cli.clj similarity index 96% rename from bb/bb/cli.clj rename to bb/cli.clj index be7ecbf..3f58652 100644 --- a/bb/bb/cli.clj +++ b/bb/cli.clj @@ -41,9 +41,9 @@ (println (c/cyan " accepts no command line arguments."))) (System/exit 0))) -(defn ->ask [{:keys [id msg prompt choices] :as _option}] +(defn ->ask [{:keys [id title prompt choices] :as _option}] {:id id - :msg msg + :msg title :type prompt :choices (if (delay? choices) @choices choices)}) @@ -73,8 +73,10 @@ (System/exit 1)) asked-opts (into {} (for [hybrid-option missing-and-askable] (println "todo: ask (menu-ask hybrid-option)" (pr-str hybrid-option)))) - cli (assoc (merge parsed-opts asked-opts) :args arguments)] - (ask-unknown! cli opts))) + cli (assoc (merge parsed-opts asked-opts) :args arguments) + out (ask-unknown! cli opts)] + (println out) + out)) (defn add-parsing-for-multi [option] (if (= :multi (:prompt option)) diff --git a/bb/bb/dl_and_run.clj b/bb/dl_and_run.clj similarity index 88% rename from bb/bb/dl_and_run.clj rename to bb/dl_and_run.clj index 0533f2b..65d685a 100644 --- a/bb/bb/dl_and_run.clj +++ b/bb/dl_and_run.clj @@ -1,19 +1,18 @@ (ns bb.dl-and-run - (:require [babashka.tasks :refer [shell]] - [babashka.curl :as curl] - [bask.colors :as c] - [bb.tasks :as t] - [selmer.parser :refer [<<]] - [cheshire.core :as json] - [clojure.edn :as edn])) + (:require + [babashka.curl :as curl] + [babashka.tasks :refer [shell]] + [bask.colors :as c] + [bb.tasks :as t] + [cheshire.core :as json] + [clojure.edn :as edn] + [clojure.string :as str] + [selmer.parser :refer [<<]])) (defn- keep-first "like (fn [f coll] (first (keep f coll))) but does not do chunking." [f coll] - (reduce (fn [_ element] - (when-let [resp (f element)] (reduced resp))) - nil - coll)) + (reduce (fn [_ element] (when-let [resp (f element)] (reduced resp))) nil coll)) (defn- gh-get [url] (try (-> url @@ -79,14 +78,15 @@ (println (c/bold "Be sure to tick the *repo* permission.")) (System/exit 1)))) -(defn download-and-run-latest-jar! [{:keys [branch port socket-repl]}] +(defn download-latest-jar! [{:keys [branch]}] (let [finished (t/wait (str "Finding uberjar for branch" (c/green branch)) "📞") {artifact-id :id created-at :created_at dl-url :archive_download_url sha :head_sha - :as info} (branch->latest-artifact branch) - branch-dir (str download-dir branch)] + :as info_} (branch->latest-artifact branch) + branch-dir (str download-dir branch) + info (into (sorted-map) (assoc info_ :branch branch :branch-dir branch-dir))] (finished) (println (c/cyan "Found latest artifact!")) (println (c/magenta (str " git SHA: " (c/green sha)))) @@ -99,6 +99,7 @@ ;; - _or_ use the older artifact? (println (c/magenta (str " Download Url: " (c/green dl-url)))) (prn info) + (println "Download directory: " branch-dir) (shell (str "mkdir -p " branch-dir)) (if (= (try (edn/read-string (slurp (str branch-dir "/info.edn"))) (catch Throwable _ ::nothing-there)) @@ -109,6 +110,10 @@ (download-mb-jar! branch-dir dl-url))) (println "Artifact download complete.") (spit (str branch-dir "/info.edn") info) + info)) + +(defn run-jar! [info port socket-repl] + (let [{:keys [branch branch-dir]} info] (println "Unzipping artifact...") (try (shell {:dir branch-dir :out nil} "unzip -o metabase.zip") @@ -133,13 +138,17 @@ (t/print-env "mb" env+) (shell {:dir branch-dir :out :inherit :env env+} cmd)))) +(defn download-and-run-latest-jar! [{:keys [branch port socket-repl] :as args}] + (let [info (download-latest-jar! args)] + (run-jar! info port socket-repl))) + (def pretty {:success "✅" :skipped "⏭ī¸ " - :cancelled "⏚ī¸ " + :cancelled "⏚ī¸" :in-progress "🔄" :failure "❌"}) -(defn checks-for-branch [{branch :branch}] +(defn checks-for-branch [branch] ;; note: this is a ref, so it can e.g. also be a sha. (->> (str "https://api.github.com/repos/metabase/metabase/commits/" branch "/check-runs") gh-get diff --git a/bb/bb/meta.clj b/bb/meta.clj similarity index 90% rename from bb/bb/meta.clj rename to bb/meta.clj index e9a41e0..dd51e42 100644 --- a/bb/bb/meta.clj +++ b/bb/meta.clj @@ -22,18 +22,20 @@ (println (c/green "[bb metabuild] ✅ Done.")))) (defn build [app-db user-name password extensions] - (let [env+ (assoc (t/env) "MB_DB_CONNECTION_URI" + (let [env+ (assoc (t/env) + "MB_DB_CONNECTION_URI" (or (t/env "FORCE_MB_DB_CONNECTION_URI" (constantly false)) (case app-db - "postgres" (str "postgres://" user-name ":" password "@localhost:5432/metabase") "mysql" (str "mysql://" user-name ":" password "@localhost:3306/metabase_test") - "h2" "" ))) + "postgres" (str "postgres://" user-name ":" password "@localhost:5432/metabase") + "h2" "" )) + "MB_DB_TYPE" app-db) cmd (str "clj -M" (str/join (map (fn [s-or-kw] (keyword (name s-or-kw))) extensions)))] (println (:out (shell {:out :string} "java -version"))) (t/print-env "mb" env+) (println (c/green "\n ==== Starting Metabase with: ==== \n")) (println (c/green cmd)) (println (c/green "\n ================================= \n")) - (println (c/magenta "In directory: " (t/env "MB_DIR"))) + (println (c/magenta "In directory 📂:" (t/env "MB_DIR"))) (future (listen-for-nrepl-and-init!)) (shell {:extra-env env+ :dir (t/env "MB_DIR")} cmd))) diff --git a/bb/bb/quick_test.clj b/bb/quick_test.clj similarity index 100% rename from bb/bb/quick_test.clj rename to bb/quick_test.clj diff --git a/bb/retry.clj b/bb/retry.clj new file mode 100644 index 0000000..5b8b05d --- /dev/null +++ b/bb/retry.clj @@ -0,0 +1,17 @@ +(ns bb.retry) + +(defn retry + "tries (thunk) until `retries` times or it returns a non-nil value" + [retries thunk] + (loop [count 0] + (let [wait (int (dec (Math/pow 2 count)))] + (Thread/sleep (* 1000 wait)) + (let [result (thunk)] + (cond (some? result) + result + (< count retries) + (do + (println (format "Retry %d/%d returned nil, retrying in %d seconds..." count (inc retries) wait)) + (recur (inc count))) + :else + (println "No more retries left")))))) diff --git a/bb/bb/tasks.clj b/bb/tasks.clj similarity index 97% rename from bb/bb/tasks.clj rename to bb/tasks.clj index 6f9bd82..2153c19 100644 --- a/bb/bb/tasks.clj +++ b/bb/tasks.clj @@ -35,7 +35,7 @@ (defn list-branches [mb-dir] (git-fetch mb-dir) - (letfn [(remove-origin [b] (str/replace (str/trim b) (re-pattern "^origin/") ""))] + (let [remove-origin (fn remove-origin [b] (str/replace (str/trim b) (re-pattern "^origin/") ""))] (mapv remove-origin (str/split-lines (->> "git branch -r" (shell {:dir mb-dir :out :string}) :out))))) diff --git a/watch_ci.clj b/watch_ci.clj deleted file mode 100755 index 879dc8c..0000000 --- a/watch_ci.clj +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bb - -(ns watch-ci - (:require [babashka.curl :as curl] - [babashka.tasks :refer [shell]] - [bask.colors :as c] - [bb.cli :as cli] - [cheshire.core :as json] - [clojure.edn :as edn] - [clojure.pprint :as pp] - [clojure.string :as str] - [selmer.parser :refer [<<]])) - -(defn env - ([] (into {} (System/getenv))) - ([env-var] (env env-var (fn [] (println "Warning: cannot find " (c/red env-var) " in env.")))) - ([env-var error-thunk] (or ((env) (name env-var)) (error-thunk)))) - -(defn- gh-GET [url] - (try (-> url - (curl/get {:headers {"Accept" "application/vnd.github+json" "Authorization" (str "Bearer " (env "GH_TOKEN"))}}) - :body - (json/decode true)) - (catch Exception e - (let [{:keys [status]} (ex-data e)] - (when (= status 401) (println (c/red "Is your GH_TOKEN out of date?"))) - (throw (ex-info (str "Error trying to get url " url " status: " status) - {:status status :url url})))))) - -(def mb-dir (env "MB_DIR" (fn [] - (println (c/red "Please set MB_DIR env variable to your metabase directory!")) - (System/exit 1)))) - -(defn current-branch [] - (->> "git rev-parse --abbrev-ref HEAD" - (shell {:dir mb-dir :out :string}) - :out - str/trim)) - -(def branch (current-branch)) - -(defn checks-for-branch - ;; note: this is a ref, so it can e.g. also be a sha. - [] - (->> (str "https://api.github.com/repos/metabase/metabase/commits/" branch "/check-runs") - gh-GET - :check_runs - (mapv (comp (fn [x] (if (nil? x) :in-progress (keyword x))) :conclusion)) - frequencies - (sort-by first) - reverse)) - -(def pretty {:success "✅" :skipped "⏭ī¸ " :cancelled "⏚ī¸" :in-progress "🔄" :failure "❌"}) - -(defn print-report-line [checks] - (print (str "[" (.format (java.text.SimpleDateFormat. "hh:mm:ss a") (java.util.Date.)) "] ")) - (doseq [[status count] checks] - (print (str/join (repeat count (pretty status)))) (print " ")) - (println) - (flush)) - -(defn -main [] - ;; (prn (env)) - (println "Legend:" (pr-str pretty)) - (loop [n 0] - (let [checks (checks-for-branch)] - (when (zero? (mod n 20)) - (println (c/green "Checking CI for branch:") (c/white branch) (c/green "."))) - (print-report-line checks) - (when-not (= (keys checks) [:success]) - (Thread/sleep 5000) - (recur (inc n)))))) - -(when (= *file* (System/getProperty "babashka.file")) (-main)) From 6441d46138092244f655c66ba1bd1749fc4d0a46 Mon Sep 17 00:00:00 2001 From: Bryan Maass Date: Thu, 2 Nov 2023 12:11:19 -0600 Subject: [PATCH 2/2] remove unused ns --- bb/retry.clj | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 bb/retry.clj diff --git a/bb/retry.clj b/bb/retry.clj deleted file mode 100644 index 5b8b05d..0000000 --- a/bb/retry.clj +++ /dev/null @@ -1,17 +0,0 @@ -(ns bb.retry) - -(defn retry - "tries (thunk) until `retries` times or it returns a non-nil value" - [retries thunk] - (loop [count 0] - (let [wait (int (dec (Math/pow 2 count)))] - (Thread/sleep (* 1000 wait)) - (let [result (thunk)] - (cond (some? result) - result - (< count retries) - (do - (println (format "Retry %d/%d returned nil, retrying in %d seconds..." count (inc retries) wait)) - (recur (inc count))) - :else - (println "No more retries left"))))))