Fix dev setup and tests, clean up

Seems things were in a fairly broken state, and given that several people have shown interest to start contributing to this app I wanted to get things in a usable state again as soon as possible, so apologies for the monster commit. This - removes Figwheel/doo/lein-cljsbuild, we don't have any CLJS so keep it simple - removes the clj/cljc/cljs directory split, again YAGNI. Easy enough to add back if needed. - Remove some more Chestnut boilerplate that doesn't apply - Makes sure `lein repl` works and you can load the demo data - Makes sure `lein test` works, this time using Kaocha - Fixes broken tests - Upgrades dependencies
clojureverse · Jul 2, 2019 · 61bf0dd · 61bf0dd
1 parent ce5fcfb
commit 61bf0dd
Show file tree

Hide file tree

Showing 39 changed files with 125 additions and 288 deletions.
diff --git a/dev/cljs/user.cljs b/dev/cljs/user.cljs
diff --git a/dev/user.clj b/dev/user.clj
@@ -2,8 +2,6 @@
   (:require [clojurians-log.application :as app]
             [clojurians-log.config :as config :refer [config]]
             [clojurians-log.repl :as repl]
-            [figwheel-sidecar.config :as fw-config]
-            [figwheel-sidecar.system :as fw-sys]
             [garden-watcher.core :refer [new-garden-watcher]]
             [lambdaisland.repl-tools.browse-url :as browse-url]
             [lambdaisland.repl-tools.ring-history :as ring-history]
@@ -19,17 +17,12 @@
     (alter-var-root #'app/config (constantly config))
     (-> (app/prod-system config)
         (ring-history/inject-ring-history)
-        (assoc :middleware (new-middleware {:middleware (clojurians-log.config/middleware-stack :dev)})
-               :figwheel-system (fw-sys/figwheel-system (fw-config/fetch-config))
-               :css-watcher (fw-sys/css-watcher {:watch-paths ["resources/public/css"]})
-               :garden-watcher (new-garden-watcher ['clojurians-log.styles])
-               #_#_:browse-url (browse-url/new-browse-url-component (str "http://localhost:" (get-in config [:http :port])))))))
+        (assoc :garden-watcher (new-garden-watcher ['clojurians-log.styles])
+               ;;:browse-url (browse-url/new-browse-url-component (str "http://localhost:" (get-in config [:http :port])))
+               ))))
 
 (reloaded.repl/set-init! #(dev-system))
 
-(defn cljs-repl []
-  (fw-sys/cljs-repl (:figwheel-system system)))
-
 ;; Set up aliases so they don't accidentally
 ;; get scrubbed from the namespace declaration
 (def start reloaded.repl/start)

diff --git a/project.clj b/project.clj
@@ -4,156 +4,78 @@
   :license {:name "Mozilla Public License 2.0"
             :url "https://www.mozilla.org/en-US/MPL/2.0/"}
 
-  :dependencies [[org.clojure/clojure "1.9.0"]
+  :dependencies [[org.clojure/clojure "1.10.1"]
                  [org.clojure/clojurescript "1.9.946" :scope "provided"]
-                 [com.cognitect/transit-clj "0.8.300"]
-                 [ring "1.6.3"]
-                 [ring/ring-defaults "0.3.1"]
-                 [bk/ring-gzip "0.2.1"]
+                 [com.cognitect/transit-clj "0.8.313"]
+                 [ring "1.7.1"]
+                 [ring/ring-defaults "0.3.2"]
+                 [bk/ring-gzip "0.3.0"]
                  [radicalzephyr/ring.middleware.logger "0.6.0"]
                  [clj-logging-config "1.9.12"]
-                 [compojure "1.6.0"]
+                 [compojure "1.6.1"]
                  [environ "1.1.0"]
-                 [com.stuartsierra/component "0.3.2"]
-                 [org.danielsz/system "0.4.2-SNAPSHOT"]
-                 [org.clojure/tools.namespace "0.3.0-alpha4"]
-                 [http-kit "2.3.0-alpha4"]
-                 [re-frame "0.10.3-SNAPSHOT"]
-                 [lambdaisland/garden-watcher "0.3.2"]
+                 [com.stuartsierra/component "0.4.0"]
+                 [org.danielsz/system "0.4.3"]
+                 [org.clojure/tools.namespace "0.3.0"]
+                 [http-kit "2.3.0"]
+                 [lambdaisland/garden-watcher "0.3.3"]
                  [hiccup "2.0.0-alpha1"]
                  [org.clojure/data.json "0.2.6"]
-                 [clojure.java-time "0.3.1"]
-                 [prone "1.1.4"]
+                 [clojure.java-time "0.3.2"]
+                 [prone "1.6.4"]
                  [com.google.guava/guava "23.5-jre"]
-                 [aero "1.1.2"]
+                 [aero "1.1.3"]
                  [lambdaisland/repl-tools "0.1.0"]
-                 [clj-http "3.7.0"]
-                 [org.julienxx/clj-slack "0.5.5"]
+                 [clj-http "3.10.0"]
+                 [org.julienxx/clj-slack "0.6.3"]
                  [reloaded.repl "0.2.4"]
-                 [instaparse "1.4.8"]
+                 [instaparse "1.4.10"]
                  [com.cemerick/url "0.1.1"]
                  [enlive "1.1.6"]
-                 [bidi "2.1.3"]]
+                 [bidi "2.1.6"]]
 
-  :plugins [[lein-cljsbuild "1.1.7"]
-            [lein-environ "1.1.0"]]
+  :plugins [[lein-environ "1.1.0"]]
 
   :min-lein-version "2.6.1"
 
-  :source-paths ["src/clj" "src/cljs" "src/cljc"]
+  :source-paths ["src"]
 
-  :test-paths ["test/clj" "test/cljc"]
-
-  :clean-targets ^{:protect false} [:target-path :compile-path "resources/public/js" "dev-target"]
+  :test-paths ["test"]
 
   :uberjar-name "clojurians-log.jar"
 
-  ;; Use `lein run` if you just want to start a HTTP server, without figwheel
   :main clojurians-log.application
 
   ;; nREPL by default starts in the :main namespace, we want to start in `user`
   ;; because that's where our development helper functions like (go) and
   ;; (browser-repl) live.
   :repl-options {:init-ns user}
 
-  :cljsbuild {:builds
-              [{:id "app"
-                :source-paths ["src/cljs" "src/cljc" "dev"]
-
-                :figwheel {:on-jsload "clojurians-log.system/reset"}
-
-                :compiler {:main cljs.user
-                           :asset-path "js/compiled/out"
-                           :output-to "dev-target/public/js/compiled/clojurians_log.js"
-                           :output-dir "dev-target/public/js/compiled/out"
-                           :source-map-timestamp true}}
-
-               {:id "test"
-                :source-paths ["src/cljs" "test/cljs" "src/cljc" "test/cljc"]
-                :compiler {:output-to "dev-target/public/js/compiled/testable.js"
-                           :main clojurians-log.test-runner
-                           :optimizations :none}}
-
-               {:id "min"
-                :source-paths ["src/cljs" "src/cljc"]
-                :jar true
-                :compiler {:main clojurians-log.system
-                           :output-to "resources/public/js/compiled/clojurians_log.js"
-                           :output-dir "target"
-                           :source-map-timestamp true
-                           :optimizations :advanced
-                           :closure-defines {goog.DEBUG false}
-                           :pretty-print false}}]}
-
-  ;; When running figwheel from nREPL, figwheel will read this configuration
-  ;; stanza, but it will read it without passing through leiningen's profile
-  ;; merging. So don't put a :figwheel section under the :dev profile, it will
-  ;; not be picked up, instead configure figwheel here on the top level.
-
-  :figwheel {;; :http-server-root "public"       ;; serve static assets from resources/public/
-              :server-port 3459                ;; default
-             ;; :server-ip "127.0.0.1"           ;; default
-             :css-dirs ["resources/public/css"]  ;; watch and update CSS
-
-             ;; Start an nREPL server into the running figwheel process. We
-             ;; don't do this, instead we do the opposite, running figwheel from
-             ;; an nREPL process, see
-             ;; https://github.com/bhauman/lein-figwheel/wiki/Using-the-Figwheel-REPL-within-NRepl
-             ;; :nrepl-port 7888
-
-             ;; To be able to open files in your editor from the heads up display
-             ;; you will need to put a script on your path.
-             ;; that script will have to take a file path and a line number
-             ;; ie. in  ~/bin/myfile-opener
-             ;; #! /bin/sh
-             ;; emacsclient -n +$2 $1
-             ;;
-             ;; :open-file-command "myfile-opener"
-
-             :server-logfile "log/figwheel.log"}
-
-  :doo {:build "test"}
-
-  :aliases {"prep" ["do"
-                    #_"compile"
-                    #_["cljsbuild" "once" "min"]
-                    ["run" "-m" "garden-watcher.main" "clojurians-log.styles"]]}
+  :aliases {"garden" ["run" "-m" "garden-watcher.main" "clojurians-log.styles"]
+            "test" ["run" "-m" "kaocha.runner"]}
 
   :profiles {:dev
-             {:dependencies [[figwheel "0.5.15-SNAPSHOT"]
-                             [figwheel-sidecar "0.5.15-SNAPSHOT"]
-                             [com.cemerick/piggieback "0.2.2"]
-                             [org.clojure/tools.nrepl "0.2.13"]
-                             [lein-doo "0.1.8"]
-                             [com.datomic/datomic-free "0.9.5656"]
-                             [com.cemerick/pomegranate "1.0.0"]
-                             [vvvvalvalval/scope-capture "0.1.4"]
+             {:dependencies [[com.datomic/datomic-free "0.9.5697" :exclusions [org.clojure/tools.cli]]
+                             [com.cemerick/pomegranate "1.1.0"]
+                             [vvvvalvalval/scope-capture "0.3.2"]
                              [alembic "0.3.2"]
-                             [cheshire "5.8.0"]
-                             [ring/ring-mock "0.3.2"]
-                             [hickory "0.7.1"]]
-
-              :plugins [[lein-figwheel "0.5.15-SNAPSHOT"]
-                        [lein-doo "0.1.8"]
-                        [com.jakemccrary/lein-test-refresh "0.22.0"]]
+                             [cheshire "5.8.1"]
+                             [ring/ring-mock "0.4.0"]
+                             [hickory "0.7.1"]
+                             [lambdaisland/kaocha "RELEASE"]]
 
-              :source-paths ["dev"]
-              :repl-options {:nrepl-middleware [cemerick.piggieback/wrap-cljs-repl]}}
+              :source-paths ["dev"]}
 
              :production
              {:aot :all
-              :source-paths ^:replace ["src/clj" "src/cljc"]
               :dependencies [;; Set through .lein/profiles.clj, see ansible scripts
                              ;; [com.datomic/datomic-pro ""]
 
                              ;; JDBC driver for Datomic+PostgreSQL
-                             [org.postgresql/postgresql "9.3-1102-jdbc41"]]}
+                             [org.postgresql/postgresql "42.2.6"]]}
 
              :uberjar
-             {:source-paths ^:replace ["src/clj" "src/cljc"]
-              :prep-tasks ["compile"
-                           ["cljsbuild" "once" "min"]
-                           ["run" "-m" "garden-watcher.main" "clojurians-log.styles"]]
+             {:prep-tasks ["compile" ["run" "-m" "garden-watcher.main" "clojurians-log.styles"]]
               :hooks []
               :omit-source true
               :aot :all}})
diff --git a/src/clj/clojurians_log/slack_api.clj b/src/clj/clojurians_log/slack_api.clj
diff --git a/src/cljc/.gitkeep b/src/cljc/.gitkeep
diff --git a/src/cljs/.gitkeep b/src/cljs/.gitkeep
diff --git a/src/clj/clojurians_log/application.clj → src/clojurians_log/application.clj b/src/clj/clojurians_log/application.clj → src/clojurians_log/application.clj
@@ -30,7 +30,7 @@
                                    (fn [request]
                                      ((home-routes endpoint) request))))
                    (component/using [:datomic :config]))
-   :middleware (new-middleware {:middleware (clojurians-log.config/middleware-stack :prod)})
+   :middleware (new-middleware {:middleware clojurians-log.config/middleware-stack})
    :handler    (-> (new-handler)
                    (component/using [:routes :middleware]))
    :http       (-> (new-web-server (:port http))

diff --git a/...jurians_log/components/datomic_schema.clj → ...jurians_log/components/datomic_schema.clj b/...jurians_log/components/datomic_schema.clj → ...jurians_log/components/datomic_schema.clj
diff --git a/...clojurians_log/components/server_info.clj → ...clojurians_log/components/server_info.clj b/...clojurians_log/components/server_info.clj → ...clojurians_log/components/server_info.clj
diff --git a/src/clj/clojurians_log/config.clj → src/clojurians_log/config.clj b/src/clj/clojurians_log/config.clj → src/clojurians_log/config.clj
@@ -28,15 +28,10 @@
                :default-charset "utf-8"}})
 
 (def middleware-stack
-  {:prod [[wrap-defaults site-defaults]
-          wrap-with-logger
-          wrap-gzip
-          prone/wrap-exceptions]
-   :dev [[wrap-file "dev-target/public"]
-         [wrap-defaults site-defaults]
-         wrap-with-logger
-         wrap-gzip
-         prone/wrap-exceptions]})
+  [[wrap-defaults site-defaults]
+   wrap-with-logger
+   wrap-gzip
+   prone/wrap-exceptions])
 
 (defn config
   ([file profile]

diff --git a/src/clj/clojurians_log/data.clj → src/clojurians_log/data.clj b/src/clj/clojurians_log/data.clj → src/clojurians_log/data.clj
diff --git a/src/clj/clojurians_log/db/import.clj → src/clojurians_log/db/import.clj b/src/clj/clojurians_log/db/import.clj → src/clojurians_log/db/import.clj
@@ -25,18 +25,22 @@
 
 (defmethod event->tx nil [{:keys [ts text channel user thread_ts] :as message}]
   (when-not (= \D (first channel)) ;; ignore direct messages
-    (let [inst (time-util/ts->inst ts)]
-      (cond-> #:message {:key (message-key message)
-                         :ts ts
-                         #_#_:inst (jt/to-java-date inst)
-                         :day (time-util/format-inst-day inst)
-                         :text text
-                         :channel [:channel/slack-id channel]
-                         :user [:user/slack-id user]}
+    (let [inst    (time-util/ts->inst ts)
+          message #:message {:key      (message-key message)
+                             :ts       ts
+                             #_#_:inst (jt/to-java-date inst)
+                             :day      (time-util/format-inst-day inst)
+                             :text     text
+                             :channel  [:channel/slack-id channel]
+                             :user     [:user/slack-id user]}]
 
-              (not (nil? thread_ts))
-              (merge #:message {:thread-ts thread_ts
-                                :thread-inst (jt/to-java-date (time-util/ts->inst thread_ts))})))))
+      (if (nil? thread_ts)
+        message
+        (let [thread-inst (time-util/ts->inst thread_ts)]
+          (merge message
+                 #:message{:thread-ts   thread_ts
+                           :thread-inst (jt/to-java-date thread-inst)
+                           :day         (time-util/format-inst-day thread-inst)}))))))
 
 (defmethod event->tx "message_deleted" [{:keys [deleted_ts channel] :as message}]
   [:db.fn/retractEntity [:message/key (message-key {:channel channel :ts deleted_ts})]])

diff --git a/src/clj/clojurians_log/db/queries.clj → src/clojurians_log/db/queries.clj b/src/clj/clojurians_log/db/queries.clj → src/clojurians_log/db/queries.clj
@@ -131,6 +131,7 @@
                [?chan :channel/slack-id ?slack-id]]
              db)))
 
+#_
 (doseq [v [#'clojurians-log.db.queries/user-names
            #'clojurians-log.db.queries/channel-thread-messages-of-day
            #'clojurians-log.db.queries/channel

diff --git a/src/clj/clojurians_log/db/schema.clj → src/clojurians_log/db/schema.clj b/src/clj/clojurians_log/db/schema.clj → src/clojurians_log/db/schema.clj
diff --git a/src/clj/clojurians_log/handlers.clj → src/clojurians_log/handlers.clj b/src/clj/clojurians_log/handlers.clj → src/clojurians_log/handlers.clj
diff --git a/src/clj/clojurians_log/message_parser.clj → src/clojurians_log/message_parser.clj b/src/clj/clojurians_log/message_parser.clj → src/clojurians_log/message_parser.clj
diff --git a/src/clj/clojurians_log/repl.clj → src/clojurians_log/repl.clj b/src/clj/clojurians_log/repl.clj → src/clojurians_log/repl.clj
@@ -56,7 +56,11 @@
         (remove #(.isDirectory %))
         sort)))
 
-(defn load-log-file! [file]
+(defn load-log-file!
+  "Import a single log file. This assumes all channels and users referenced in the
+  log file already exist, either by importing directly from Slack (production)
+  or from EDN files (demo data)."
+  [file]
   (println (str file))
   (let [msgs (filter #(= (:type %) "message") (data/event-seq file))
         events (keep import/event->tx msgs)]
@@ -78,40 +82,21 @@
                        (partition-all 1000))]
       @(d/transact (conn) users)))
   @(d/transact (conn) (edn/read-string (slurp (str directory "/channels.edn"))))
-  (run! load-log-file! (log-files directory)))
+  (run! load-log-file! (log-files (java.io.File. directory "logs"))))
 
-(defn load-demo-data2!
-  "Load the demo data (users, channels, messages)."
-  [directory]
-  (if-not (conn)
-    (println "Can't find Datomic connection. Make sure the system is up and running with (user/go).")
-    (do
-      (println "Importing users.edn")
-      (doseq [users (->> "/users.edn"
-                         (str directory)
-                         read-edn
-                         (partition-all 1000))]
-        (slack/import-users! (conn) users))
-
-      (println "Importing channels.edn")
-      (slack/import-channels! (conn) (->> "/channels.edn"
-                                          (str directory)
-                                          read-edn))
-
-      (println "Import messages")
-      (run! load-log-file! (log-files directory)))))
-
-(defn load-from [date]
+(defn load-from
+  "Load log files starting from a certain date (a string like \"2019-05-20\")"
+  [date]
   (->> (log-files)
-       (drop-while #(not (str/starts-with? (.getName %) date)))
+       (drop-while #(not (clojure.string/starts-with? (.getName %) date)))
        (run! load-log-file!)))
 
 (comment
   ;; rlwrap nc localhost 50505
   (use 'clojurians-log.repl)
   (load-slack-data!)
-  (load-from "2016-08-04")
   (run! load-log-file! (log-files))
+  (load-from "2016-08-04")
 
 
 

diff --git a/src/clj/clojurians_log/response.clj → src/clojurians_log/response.clj b/src/clj/clojurians_log/response.clj → src/clojurians_log/response.clj
diff --git a/src/clj/clojurians_log/routes.clj → src/clojurians_log/routes.clj b/src/clj/clojurians_log/routes.clj → src/clojurians_log/routes.clj
diff --git a/src/clj/clojurians_log/routes_def.clj → src/clojurians_log/routes_def.clj b/src/clj/clojurians_log/routes_def.clj → src/clojurians_log/routes_def.clj
@@ -8,6 +8,7 @@
 (def routes
   ["/" {"healthcheck" (-> (fn [endpoint req]
                             {:headers {"Content-Type" "text/plain"}
+                             :status 200
                              :body "OK"})
                           (bidi/tag :health-check))