Script updates (#1768)

* cleaner main function invocation for build/deploy static assets bb scripts

Uses trick similar to python standard practice to only run main function when being called as a script, improving REPL interaction

* add bin scripts for managing whitelists

* move math/bin scripts to main bin dir

specifically, close-conv.clj and purge-pii.clj

* remove old/stubbed math/bin scripts

* remove close-conv script (can do this from admin UI now)

* clean up bin/purge-pii.clj usng bin/lib and -main fn

bin/activate-xid-whitelist.clj

@@ -0,0 +1,25 @@
+#!/usr/bin/env -S bb --classpath bin
+
+(require '[lib.db :as db]
+         '[clojure.java.io :as io]
+         '[clojure.string :as str]
+         '[honey.sql.helpers :as sqlh])
+
+(defn resolve-zid [{:as opts-map :strs [--zid --zinvite]}]
+  (if --zid
+    (Integer/parseInt --zid)
+    (db/get-zinvite-zid --zinvite)))
+
+(defn -main [& {:as opts-map :strs [--zid --zinvite]}]
+  (let [zid (resolve-zid opts-map)]
+    (db/execute!
+      (-> (sqlh/update :conversations)
+          (sqlh/set {:use_xid_whitelist true})
+          (sqlh/where [:= :zid zid])))
+    (println "Done")))
+
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (apply -main *command-line-args*))
+
+

bin/add-xid-whitelist.clj

@@ -0,0 +1,41 @@
+#!/usr/bin/env -S bb --classpath bin
+
+(require '[lib.db :as db]
+         '[clojure.java.io :as io]
+         '[clojure.string :as str])
+
+(def batch-size 50)
+
+(defn xid-seq [filename]
+  (map
+    (fn [line] (first (str/split line #",")))
+    (line-seq (io/reader filename))))
+
+(defn xid-record [owner xid]
+  {:owner owner
+   :xid xid})
+
+(defn resolve-owner-id [{:as opts-map :strs [--owner --owner-email]}]
+  (if --owner
+    (Integer/parseInt --owner)
+    (db/get-email-uid --owner-email)))
+
+(defn -main [& {:as opts-map :strs [--owner --owner-email --xid-file]}]
+  (let [xids (xid-seq --xid-file)
+        owner-id (resolve-owner-id opts-map)]
+    (loop [xids-batch (take batch-size xids)
+           xids-rest (drop batch-size xids)]
+      (when (seq xids-rest)
+        (db/upsert! :xid_whitelist
+                    :xid_whitelist_owner_xid_key
+                    (map (partial xid-record owner-id)
+                         xids-batch))
+        (recur (take batch-size xids-rest)
+               (drop batch-size xids-rest))))
+    (println "Done")))
+
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (apply -main *command-line-args*))
+
+

bin/build-static-assets.clj

@@ -97,17 +97,20 @@
   (monitor-execution (clean-containers client-dir)))
   ;; once that has completed, 
 
-(def processes
-  (for [client-dir ["client-admin" "client-participation"]];; "client-report"]] ; leaving client-report off for now
-    (async/thread
-      (build-and-cp-client client-dir)
-      (println "Finished building:" client-dir))))
-
-;; Initiate all of the processes, since for is a lazy list
-(doall processes)
-;; for each process, wait until the process completes
-(doseq [proc processes]
-  (async/<!! proc))
 
-;; QED
+(defn -main []
+  (let [processes
+        (for [client-dir ["client-admin" "client-participation"]];; "client-report"]] ; leaving client-report off for now
+          (async/thread
+            (build-and-cp-client client-dir)
+            (println "Finished building:" client-dir)))]
+    ;; Initiate all of the processes, since for is a lazy list
+    (doall processes)
+    ;; for each process, wait until the process completes
+    (doseq [proc processes]
+      (async/<!! proc))))
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (apply -main *command-line-args*))
 
+;; QED

bin/deploy-static-assets.clj

@@ -161,7 +161,8 @@
       (println "Deploy completed without error."))))
 
 
-(apply -main *command-line-args*)
+(when (= *file* (System/getProperty "babashka.file"))
+  (apply -main *command-line-args*))
 
 ;; QED
 

bin/lib/db.clj

@@ -0,0 +1,71 @@
+(ns lib.db
+  "Database utilities for clj bin/ scripts"
+  (:require [babashka.pods :as pods]
+            [babashka.deps :as deps]
+            [clojure.pprint :as pp]
+            [clojure.tools.cli :as cli]
+            [clojure.java.io :as io]
+            [clojure.string :as string]))
+
+(pods/load-pod 'org.babashka/postgresql "0.0.1")
+(deps/add-deps '{:deps {com.github.seancorfield/honeysql {:mvn/version "2.5.1103"}}})
+
+(require '[pod.babashka.postgresql :as pg]
+         '[honey.sql :as sql]
+         '[honey.sql.helpers :as sqlh])
+
+
+(def db-url
+  (System/getenv "DATABASE_URL"))
+
+(defn heroku-url-spec [db-url]
+  (let [[_ user password host port db] (re-matches #"postgres://(?:(.+):(.*)@)?([^:]+)(?::(\d+))?/(.+)" db-url)]
+    {:dbtype "postgresql"
+     :host host
+     :dbname db
+     :port (or port 80)
+     :user user
+     :password password}))
+
+(defn execute-sql! [args]
+  (println "Executing sql:" args)
+  (pg/execute!
+    (heroku-url-spec (System/getenv "DATABASE_URL"))
+    args))
+
+(defn execute!
+  [query-or-command]
+  (execute-sql! (sql/format query-or-command)))
+
+
+(defn insert!
+  [table values]
+  (execute! {:insert-into table
+             :values values}))
+
+(defn upsert!
+  [table constraint values]
+  (sqlh/on-constraint :xid_whitelist_owner_xid_key)
+  (execute!
+    {:insert-into table
+     :values values
+     :on-conflict {:on-constraint constraint}
+     :do-nothing true}))
+
+
+(defn get-email-uid [email]
+  (:users/uid
+    (first
+      (execute!
+        {:select [:*]
+         :from [:users]
+         :where [:= :email email]}))))
+
+(defn get-zinvite-zid [zinvite]
+  (:zinvites/zid
+    (first
+      (execute!
+        {:select [:*]
+         :from [:zinvites]
+         :where [:= :zinvite zinvite]}))))
+

bin/purge-pii.clj

@@ -0,0 +1,31 @@
+#!/usr/bin/env -S bb --classpath bin
+
+(require '[lib.db :as db]
+         '[clojure.pprint :as pp]
+         '[honey.sql.helpers :as sqlh])
+
+;; Should we delete their conversations as well?
+;; assuming leave IP address
+;; Should maybe make sure we don't have their IP address from before we started saving encrypted
+
+(defn expunge-record
+  [table where attrs]
+  (db/execute!
+    {:update table
+     :where where
+     :set (into {} (map #(vector %1 nil) attrs))}))
+
+(defn -main [email]
+  (if-let [uid (db/get-email-uid email)]
+    (do
+      (println "Found uid:" uid)
+      (println "Deleting user data:")
+      (expunge-record :users [:= :uid uid] [:hname :email])
+      (println "Deleting participants_extended data:")
+      (expunge-record :participants_extended [:= :uid uid] [:subscribe_email]))
+    (println "Could not find uid for email address")))
+
+(when (= *file* (System/getProperty "babashka.file"))
+  (apply -main *command-line-args*))
+
+