update workflow; use core as the new namespace

README.md

@@ -15,11 +15,11 @@ Cumulo Reel
 Server side:
 
 ```clojure
-(defonce *reel (atom (merge cumulo-reel.reel/reel-schema {:base initial-db, :db initial-db})))
+(defonce *reel (atom (merge cumulo-reel.core/reel-schema {:base initial-db, :db initial-db})))
 
-(cumulo-reel.reel/reel-reducer @*reel updater op op-data sid op-id op-time)
+(cumulo-reel.core/reel-reducer @*reel updater op op-data sid op-id op-time)
 
-(reset! *reel (cumulo-reel.reel/refresh-reel @*reel initial-db updater))
+(reset! *reel (cumulo-reel.core/refresh-reel @*reel initial-db updater))
 ```
 
 Client side:

cli/build/main.clj

@@ -5,23 +5,23 @@
 
 (defn sh! [command]
   (println command)
-  (println (sh "bash" "-c" command)))
+  (println (:out (sh "bash" "-c" command))))
 
-(defn build []
+(defn build-cdn []
   (sh! "rm -rf dist/*")
   (shadow/release :client)
   (shadow/release :server)
   (shadow/compile :page)
-  (sh! "mode=release node target/page.js")
+  (sh! "release=true cdn=true node target/page.js")
   (sh! "cp package.json dist/")
   (sh! "cp entry/manifest.json dist/"))
 
-(defn build-local []
+(defn build []
   (sh! "rm -rf dist/*")
   (shadow/release :client)
   (shadow/release :server)
   (shadow/compile :page)
-  (sh! "mode=local-bundle node target/page.js")
+  (sh! "release=true node target/page.js")
   (sh! "cp package.json dist/")
   (sh! "cp entry/manifest.json dist/"))
 

package.json

@@ -5,10 +5,11 @@
   "main": "index.js",
   "scripts": {
     "watch": "shadow-cljs watch client server",
+    "build-cdn": "shadow-cljs clj-run build.main/build-cdn",
     "build": "shadow-cljs clj-run build.main/build",
-    "build-local": "shadow-cljs clj-run build.main/build-local",
     "page": "shadow-cljs clj-run build.main/page",
     "upload": "lumo -c cli/:src/ -m build.upload",
+    "up": "yarn build-cdn && yarn upload",
     "serve": "http-server dist -s",
     "repl": "rlwrap shadow-cljs clj-repl client",
     "repl-server": "rlwrap shadow-cljs clj-repl server",
@@ -30,7 +31,7 @@
   },
   "devDependencies": {
     "http-server": "^0.11.1",
-    "shadow-cljs": "^2.7.6",
+    "shadow-cljs": "^2.7.9",
     "source-map-support": "^0.5.9"
   }
 }

shadow-cljs.edn

@@ -1,8 +1,10 @@
 
 {:source-paths ["src" "cli"]
  :dependencies [[mvc-works/hsl          "0.1.2"]
-                [mvc-works/shell-page   "0.1.9"]
+                [mvc-works/shell-page   "0.1.10"]
+                [mvc-works/ws-edn       "0.1.1"]
                 [cumulo/recollect       "0.5.0"]
+                [cumulo/util            "0.1.4"]
                 [respo                  "0.10.2"]
                 [respo/ui               "0.3.10"]
                 [respo/alerts           "0.3.10"]

src/cumulo_reel/client.cljs

@@ -4,9 +4,11 @@
             [respo.cursor :refer [mutate]]
             [cumulo-reel.comp.container :refer [comp-container]]
             [cljs.reader :refer [read-string]]
-            [cumulo-reel.connection :refer [send! setup-socket!]]
             [cumulo-reel.schema :as schema]
-            [cumulo-reel.config :as config]))
+            [cumulo-reel.config :as config]
+            [ws-edn.client :refer [ws-connect! ws-send!]]
+            [recollect.patch :refer [patch-twig]])
+  (:require-macros [clojure.core.strint :refer [<<]]))
 
 (declare dispatch!)
 
@@ -29,14 +31,20 @@
   (case op
     :states (reset! *states ((mutate op-data) @*states))
     :effect/connect (connect!)
-    (send! op op-data)))
+    (ws-send! {:kind :op, :op op, :data op-data})))
 
 (defn connect! []
-  (setup-socket!
-   *store
-   {:url (str "ws://" (.-hostname js/location) ":" (:port config/site)),
-    :on-close! (fn [event] (reset! *store nil) (.error js/console "Lost connection!")),
-    :on-open! (fn [event] (simulate-login!))}))
+  (ws-connect!
+   (<< "ws://~{js/location.hostname}:~(:port config/site)")
+   {:on-open (fn [] (simulate-login!)),
+    :on-close (fn [event] (reset! *store nil) (js/console.error "Lost connection!")),
+    :on-data (fn [data]
+      (case (:kind data)
+        :patch
+          (let [changes (:data data)]
+            (js/console.log "Changes" (clj->js changes))
+            (reset! *store (patch-twig @*store changes)))
+        (println "unknown kind:" data)))}))
 
 (def mount-target (.querySelector js/document ".app"))
 
@@ -46,6 +54,7 @@
 (def ssr? (some? (.querySelector js/document "meta.respo-ssr")))
 
 (defn main! []
+  (println "Running mode:" (if config/dev? "dev" "release"))
   (if ssr? (render-app! realize-ssr!))
   (render-app! render!)
   (connect!)

src/cumulo_reel/config.cljs

@@ -1,16 +1,21 @@
 
-(ns cumulo-reel.config (:require [cumulo-reel.util :refer [get-env!]]))
+(ns cumulo-reel.config (:require [cumulo-util.core :refer [get-env!]]))
 
-(def bundle-builds #{"release" "local-bundle"})
+(def cdn?
+  (cond
+    (exists? js/window) false
+    (exists? js/process) (= "true" js/process.env.cdn)
+    :else false))
 
 (def dev?
-  (if (exists? js/window)
-    (do ^boolean js/goog.DEBUG)
-    (not (contains? bundle-builds (get-env! "mode")))))
+  (let [debug? (do ^boolean js/goog.DEBUG)]
+    (cond
+      (exists? js/window) debug?
+      (exists? js/process) (not= "true" js/process.env.release)
+      :else true)))
 
 (def site
-  {:storage-key "workflow-storage",
-   :port 5021,
+  {:port 5021,
    :title "Cumulo",
    :icon "http://cdn.tiye.me/logo/cumulo.png",
    :dev-ui "http://localhost:8100/main.css",
@@ -19,4 +24,6 @@
    :cdn-folder "tiye.me:cdn/cumulo-workflow",
    :upload-folder "tiye.me:repo/Cumulo/workflow/",
    :server-folder "tiye.me:servers/workflow",
-   :theme "#eeeeff"})
+   :theme "#eeeeff",
+   :storage-key "workflow-storage",
+   :storage-file "storage.edn"})

src/cumulo_reel/reel.cljs → src/cumulo_reel/core.cljs

@@ -1,5 +1,5 @@
 
-(ns cumulo-reel.reel
+(ns cumulo-reel.core
   (:require [clojure.string :as string] [cumulo-reel.config :refer [dev?]]))
 
 (defn play-records [db records updater]

src/cumulo_reel/page.cljs

@@ -6,8 +6,7 @@
             [cljs.reader :refer [read-string]]
             [cumulo-reel.schema :as schema]
             [cumulo-reel.config :as config]
-            [cumulo-reel.util :refer [get-env!]]
-            [build.util :refer [get-ip!]])
+            [cumulo-util.build :refer [get-ip!]])
   (:require-macros [clojure.core.strint :refer [<<]]))
 
 (def base-info
@@ -21,16 +20,14 @@
    ""
    (merge
     base-info
-    {:styles [(<< "http://~{(get-ip!)}:8100/main.css") "/entry/main.css"],
+    {:styles [(<< "http://~(get-ip!):8100/main.css") "/entry/main.css"],
      :scripts ["/client.js"],
      :inline-styles []})))
 
-(def local-bundle? (= "local-bundle" (get-env! "mode")))
-
 (defn prod-page []
   (let [html-content (make-string (comp-container {} nil))
         assets (read-string (slurp "dist/assets.edn"))
-        cdn (if local-bundle? "" (:cdn-url config/site))
+        cdn (if config/cdn? (:cdn-url config/site) "")
         prefix-cdn #(str cdn %)]
     (make-page
      html-content
@@ -40,6 +37,7 @@
        :scripts (map #(-> % :output-name prefix-cdn) assets)}))))
 
 (defn main! []
-  (if (contains? config/bundle-builds (get-env! "mode"))
-    (spit "dist/index.html" (prod-page))
-    (spit "target/index.html" (dev-page))))
+  (println "Running mode:" (if config/dev? "dev" "release"))
+  (if config/dev?
+    (spit "target/index.html" (dev-page))
+    (spit "dist/index.html" (prod-page))))

src/cumulo_reel/server.cljs

@@ -1,80 +1,96 @@
 
 (ns cumulo-reel.server
   (:require [cumulo-reel.schema :as schema]
-            [cumulo-reel.service :refer [run-server! sync-clients!]]
             [cumulo-reel.updater :refer [updater]]
             [cljs.reader :refer [read-string]]
-            [cumulo-reel.reel :refer [reel-reducer refresh-reel reel-schema]]
+            [cumulo-reel.core :refer [reel-reducer refresh-reel reel-schema]]
             ["fs" :as fs]
-            ["shortid" :as shortid]
-            ["child_process" :as cp]
             ["path" :as path]
-            [cumulo-reel.node-config :as node-config]
-            [cumulo-reel.node-config :refer [dev?]]
-            [cumulo-reel.config :as config]))
+            [cumulo-reel.config :as config]
+            [cumulo-util.file :refer [write-mildly! get-backup-path! merge-local-edn!]]
+            [cumulo-util.core :refer [id! repeat! unix-time! delay!]]
+            [cumulo-reel.twig.container :refer [twig-container]]
+            [recollect.diff :refer [diff-twig]]
+            [recollect.twig :refer [render-twig]]
+            [ws-edn.server :refer [wss-serve! wss-send! wss-each!]]))
 
-(def initial-db
-  (let [filepath (:storage-path node-config/env)]
-    (if (fs/existsSync filepath)
-      (do
-       (println "Found storage in:" (:storage-path node-config/env))
-       (read-string (fs/readFileSync filepath "utf8")))
-      schema/database)))
+(defonce *client-caches (atom {}))
+
+(def storage-file (path/join js/__dirname (:storage-file config/site)))
+
+(defonce initial-db
+  (merge-local-edn!
+   schema/database
+   storage-file
+   (fn [found?] (if found? (println "Found local EDN data") (println "Found no data")))))
 
 (defonce *reel (atom (merge reel-schema {:base initial-db, :db initial-db})))
 
 (defonce *reader-reel (atom @*reel))
 
-(defn detect-then-write! [file-path content]
-  (let [do-write! (fn []
-                    (cp/execSync (str "mkdir -p " (path/dirname file-path)))
-                    (fs/writeFileSync file-path content)
-                    (println "Write to file:" file-path))]
-    (if (fs/existsSync file-path)
-      (let [old-content (fs/readFileSync file-path "utf8")]
-        (if (not= content old-content)
-          (do-write!)
-          (comment println "same file, skipping:" file-path)))
-      (do-write!))))
-
 (defn persist-db! []
   (let [file-content (pr-str (assoc (:db @*reel) :sessions {}))
-        now (js/Date.)
-        storage-path (:storage-path node-config/env)
-        backup-path (path/join
-                     js/__dirname
-                     "backups"
-                     (str (inc (.getMonth now)))
-                     (str (.getDate now) "-storage.edn"))]
-    (detect-then-write! storage-path file-content)
-    (detect-then-write! backup-path file-content)))
+        storage-path storage-file
+        backup-path (get-backup-path!)]
+    (write-mildly! storage-path file-content)
+    (write-mildly! backup-path file-content)))
 
 (defn dispatch! [op op-data sid]
-  (let [op-id (.generate shortid), op-time (.valueOf (js/Date.))]
-    (if dev? (println "Dispatch!" (str op) op-data sid))
+  (let [op-id (id!), op-time (unix-time!)]
+    (if config/dev? (println "Dispatch!" (str op) op-data sid))
     (try
      (cond
        (= op :effect/persist) (persist-db!)
-       :else
-         (let [new-reel (reel-reducer @*reel updater op op-data sid op-id op-time)]
-           (reset! *reel new-reel)))
-     (catch js/Error error (.error js/console error)))))
+       :else (reset! *reel (reel-reducer @*reel updater op op-data sid op-id op-time)))
+     (catch js/Error error (js/console.error error)))))
 
 (defn on-exit! [code]
   (persist-db!)
   (comment println "exit code is:" (pr-str code))
-  (.exit js/process))
+  (js/process.exit))
+
+(defn sync-clients! [reel]
+  (wss-each!
+   (fn [sid socket]
+     (let [db (:db reel)
+           records (:records reel)
+           session (get-in db [:sessions sid])
+           old-store (or (get @*client-caches sid) nil)
+           new-store (render-twig (twig-container db session records) old-store)
+           changes (diff-twig old-store new-store {:key :id})]
+       (println "Changes for" sid ":" changes (count records))
+       (if (not= changes [])
+         (do
+          (wss-send! sid {:kind :patch, :data changes})
+          (swap! *client-caches assoc sid new-store)))))))
 
 (defn render-loop! []
-  (if (not (identical? @*reader-reel @*reel))
-    (do (reset! *reader-reel @*reel) (sync-clients! @*reader-reel)))
-  (js/setTimeout render-loop! 200))
+  (when (not (identical? @*reader-reel @*reel))
+    (reset! *reader-reel @*reel)
+    (sync-clients! @*reader-reel))
+  (delay! 0.2 render-loop!))
+
+(defn run-server! []
+  (wss-serve!
+   (:port config/site)
+   {:on-open (fn [sid socket]
+      (dispatch! :session/connect nil sid)
+      (js/console.info "New client.")),
+    :on-data (fn [sid action]
+      (case (:kind action)
+        :op (dispatch! (:op action) (:data action) sid)
+        (println "unknown data" action))),
+    :on-close (fn [sid event]
+      (js/console.warn "Client closed!")
+      (dispatch! :session/disconnect nil sid)),
+    :on-error (fn [error] (.error js/console error))}))
 
 (defn main! []
-  (run-server! #(dispatch! %1 %2 %3) (:port config/site))
+  (println "Running mode:" (if config/dev? "dev" "release"))
+  (run-server!)
   (render-loop!)
-  (.on js/process "SIGINT" on-exit!)
-  (js/setInterval #(persist-db!) (* 60 1000 10))
+  (js/process.on "SIGINT" on-exit!)
+  (repeat! 600 #(persist-db!))
   (println "Server started."))
 
 (defn reload! []