Wire up frontend to use nbb export and html

- Add tests for export
- Refactor export to be more readable
- Add error notification for export that user can see
- Remove unused :ui/show-recent? state
- Cleanup and move escape fns to proper locations

deps.edn

@@ -29,6 +29,7 @@
   org.clojars.mmb90/cljs-cache          {:mvn/version "0.1.4"}
   logseq/common                         {:local/root "deps/common"}
   logseq/graph-parser                   {:local/root "deps/graph-parser"}
+  logseq/publish-spa                    {:local/root "deps/publish-spa"}
   metosin/malli                         {:mvn/version "0.10.0"}
   fipp/fipp                             {:mvn/version "0.6.26"}}
 

deps/publish-spa/.clj-kondo/config.edn

@@ -0,0 +1,13 @@
+{:linters
+ {:aliased-namespace-symbol {:level :warning}
+  :namespace-name-mismatch {:level :warning}
+  :used-underscored-binding {:level :warning}
+
+  :consistent-alias
+  {:aliases {clojure.string string}}}
+
+ :lint-as
+ {logseq.publish-spa.test.helper/deftest-async clojure.test/deftest
+  promesa.core/let clojure.core/let}
+ :skip-comments true
+ :output {:progress true}}

deps/publish-spa/.gitignore

@@ -1,2 +1,3 @@
 /.clj-kondo/.cache
 .nbb
+/tmp

deps/publish-spa/deps.edn

@@ -0,0 +1,3 @@
+{:paths ["src"]
+ :deps
+ {logseq/db {:local/root "../db"}}}

deps/publish-spa/nbb.edn

@@ -1,4 +1,8 @@
-{:deps
+{:paths ["src"]
+ :deps
  {logseq/graph-parser
   ;; Nbb bug. Should just be "../graph-parser"
-  {:local/root "../../../../graph-parser"}}}
+  {:local/root "../../../../graph-parser"}
+  io.github.nextjournal/nbb-test-runner
+  {:git/sha "60ed57aa04bca8d604f5ba6b28848bd887109347"
+   #_#_:local/root "../../../../../../nbb-test-runner"}}}

deps/publish-spa/package.json

@@ -6,9 +6,10 @@
     "@logseq/nbb-logseq": "^1.2.168"
   },
   "dependencies": {
-    "mldoc": "^1.5.1"
+    "mldoc": "^1.5.1",
+    "fs-extra": "9.1.0"
   },
   "scripts": {
-    "test": "nbb-logseq -cp src:test -m logseq.publish-spa.nbb-test-runner/run-tests"
+    "test": "nbb-logseq -cp src:test -m nextjournal.test-runner"
   }
 }

deps/publish-spa/src/logseq/publish_spa.cljs

@@ -2,10 +2,9 @@
   (:require [datascript.transit :as dt]
             [logseq.publish-spa.html :as html]
             [logseq.publish-spa.export :as export]
-            [logseq.publish-spa.db :as db]
-            ["path" :as path]))
+            [logseq.publish-spa.db :as db]))
 
-(defn prep-for-export [db {:keys [app-state repo-config]}]
+(defn prep-for-export [db {:keys [app-state repo-config html-options]}]
   (let [[db asset-filenames']
         (if (:publishing/all-pages-public? repo-config)
           (db/clean-export! db)
@@ -14,17 +13,12 @@
         db-str (dt/write-transit-str db)
         state (assoc (select-keys app-state
                             [:ui/theme
-                             :ui/sidebar-collapsed-blocks
-                             :ui/show-recent?])
+                             :ui/sidebar-collapsed-blocks])
                      :config {"local" repo-config})
-        raw-html-str (html/publishing-html db-str (pr-str state))]
+        raw-html-str (html/publishing-html db-str state html-options)]
     {:html raw-html-str
      :asset-filenames asset-filenames}))
 
-(defn publish [db graph-dir output-path options]
-  (let [{:keys [html asset-filenames]}
-        (prep-for-export db options)
-        custom-css-path (path/join graph-dir "logseq" "custom.css")
-        export-css-path (path/join graph-dir "logseq" "export.css")
-        app-path "../../static"]
-    (export/handle-export-publish-assets html app-path custom-css-path export-css-path graph-dir asset-filenames output-path)))
+(defn publish [db static-dir graph-dir output-path options]
+  (let [{:keys [html asset-filenames]} (prep-for-export db options)]
+    (export/export html static-dir graph-dir output-path {:asset-filenames asset-filenames})))

deps/publish-spa/src/logseq/publish_spa/cli.cljs

@@ -5,15 +5,6 @@
             ["path" :as path]
             [clojure.edn :as edn]))
 
-(defn- get-repo-config
-  [dir]
-  (if (fs/existsSync (path/join dir "logseq" "config.edn"))
-    (-> (path/join dir "logseq" "config.edn") fs/readFileSync str edn/read-string)
-    {}))
-
-(defn- get-app-state
-  []
-  {:ui/theme "dark", :ui/sidebar-collapsed-blocks {}, :ui/show-recent? false})
 
 (defn- get-db [graph-dir]
   (let [{:keys [conn]} (gp-cli/parse-graph graph-dir {:verbose false})] @conn))
@@ -24,9 +15,9 @@
                        (throw (ex-info "GRAPH DIR required" {})))
         output-path (or (second args)
                         (throw (ex-info "OUT DIR required" {})))
-        repo-config (get-repo-config graph-dir)]
+        repo-config (-> (path/join graph-dir "logseq" "config.edn") fs/readFileSync str edn/read-string)]
     (publish-spa/publish (get-db graph-dir)
+                         "../../static"
                          graph-dir
                          output-path
-                         {:app-state (get-app-state)
-                          :repo-config repo-config})))
+                         {:repo-config repo-config})))

deps/publish-spa/src/logseq/publish_spa/export.cljs

@@ -1,61 +1,93 @@
 (ns logseq.publish-spa.export
-  (:require ["fs-extra$default" :as fs]
-            ["path" :as path]
+  (:require ["fs-extra" :as fse]
+            ["path" :as node-path]
+            ["fs" :as fs]
             [promesa.core :as p]))
 
-(defn handle-export-publish-assets
-  [html app-path custom-css-path export-css-path repo-path asset-filenames output-path]
-  (let [root-dir output-path
-        static-dir (path/join root-dir "static")
-        assets-from-dir (path/join repo-path "assets")
-        assets-to-dir (path/join root-dir "assets")
-        index-html-path (path/join root-dir "index.html")]
-    (p/let [_ (. fs ensureDir static-dir)
-            _ (. fs ensureDir assets-to-dir)
-            _ (p/all (concat
-                      [(. fs writeFile index-html-path html)
+(def js-files
+  "js files from publishing release build"
+  ["main.js" "code-editor.js" "excalidraw.js" "tldraw.js"])
 
+(def static-dirs
+  "dirs under static dir to copy over"
+  ["css" "fonts" "icons" "img" "js"])
 
-                       (. fs copy (path/join app-path "404.html") (path/join root-dir "404.html"))]
+(defn- default-notification
+  [msg]
+  (if (= (:type msg) "success")
+    (js/console.log (:payload msg))
+    (js/console.error (:payload msg))))
+
+(defn- cleanup-js-dir
+  "Moves used js files to the correct dir and removes unused js files"
+  [output-static-dir]
+  (let [publishing-dir (node-path/join output-static-dir "js" "publishing")]
+    (p/let [_ (p/all (map (fn [file]
+                            (fs/rmSync (node-path/join output-static-dir "js" file) #js {:force true}))
+                          js-files))
+            _ (p/all (map (fn [file]
+                            (fs/renameSync
+                             (node-path/join publishing-dir file)
+                             (node-path/join output-static-dir "js" file)))
+                          js-files))
+            ;; remove publishing-dir
+            _ (p/all (map (fn [file]
+                            (fs/rmSync (node-path/join publishing-dir file)))
+                          (fs/readdirSync publishing-dir)))
+            _ (fs/rmdirSync publishing-dir)
+            ;; remove source map files
+            _ (p/all (map (fn [file]
+                            (fs/rmSync (node-path/join output-static-dir "js" (str file ".map")) #js {:force true}))
+                          ["main.js" "code-editor.js" "excalidraw.js"]))])))
+
+(defn- copy-static-files-and-assets
+  [static-dir repo-path output-dir {:keys [log-error-fn asset-filenames]
+                                    :or {asset-filenames []
+                                         log-error-fn js/console.error}}]
+  (let [assets-from-dir (node-path/join repo-path "assets")
+        assets-to-dir (node-path/join output-dir "assets")
+        output-static-dir (node-path/join output-dir "static")]
+    (p/let [_ (fs/mkdirSync assets-to-dir #js {:recursive true})
+            _ (p/all (concat
+                      [(fse/copy (node-path/join static-dir "404.html") (node-path/join output-dir "404.html"))]
 
                       (map
                        (fn [filename]
-                         (-> (. fs copy (path/join assets-from-dir filename) (path/join assets-to-dir filename))
+                         (-> (fse/copy (node-path/join assets-from-dir filename) (node-path/join assets-to-dir filename))
                              (p/catch
                               (fn [e]
-                                ;; TODO: Make into a callback
-                                (println "Failed to copy"
-                                         (str {:from (path/join assets-from-dir filename)
-                                               :to (path/join assets-to-dir filename)})
-                                         e)))))
+                                (log-error-fn "Failed to copy"
+                                              (str {:from (node-path/join assets-from-dir filename)
+                                                    :to (node-path/join assets-to-dir filename)})
+                                              e)))))
                        asset-filenames)
 
                       (map
                        (fn [part]
-                         (. fs copy (path/join app-path part) (path/join static-dir part)))
-                       ["css" "fonts" "icons" "img" "js"])))
-            export-css (if (fs/existsSync export-css-path) (. fs readFile export-css-path) "")
-            _ (. fs writeFile (path/join static-dir "css" "export.css")  export-css)
-            custom-css (if (fs/existsSync custom-css-path) (. fs readFile custom-css-path) "")
-            _ (. fs writeFile (path/join static-dir "css" "custom.css") custom-css)
-            js-files ["main.js" "code-editor.js" "excalidraw.js" "tldraw.js"]
-            _ (p/all (map (fn [file]
-                            (. fs removeSync (path/join static-dir "js" file)))
-                          js-files))
-            _ (p/all (map (fn [file]
-                            (. fs moveSync
-                              (path/join static-dir "js" "publishing" file)
-                              (path/join static-dir "js" file)))
-                          js-files))
-            _ (. fs removeSync (path/join static-dir "js" "publishing"))
-            ;; remove source map files
-            ;; TODO: ugly, replace with ls-files and filter with ".map"
-            _ (p/all (map (fn [file]
-                            (. fs removeSync (path/join static-dir "js" (str file ".map"))))
-                          ["main.js" "code-editor.js" "excalidraw.js"]))]
-
-           ;; TODO: Make into a callback
-           (println
-            :notification
-            {:type "success"
-             :payload (str "Export public pages and publish assets to " root-dir " successfully 🎉")}))))
+                         (fse/copy (node-path/join static-dir part) (node-path/join output-static-dir part)))
+                       static-dirs)))])))
+
+(defn export
+  "Given a graph's directory, the generated html and the directory containing
+  html/static assets, creates an index.html with supporting assets at the
+  specified output directory"
+  [html static-dir repo-path output-dir {:keys [notification-fn]
+                                         :or {notification-fn default-notification}
+                                         :as options}]
+  (let [custom-css-path (node-path/join repo-path "logseq" "custom.css")
+        export-css-path (node-path/join repo-path "logseq" "export.css")
+        output-static-dir (node-path/join output-dir "static")
+        index-html-path (node-path/join output-dir "index.html")]
+    (-> (p/let [_ (fs/mkdirSync output-static-dir #js {:recursive true})
+                _ (fs/writeFileSync index-html-path html)
+                _ (copy-static-files-and-assets static-dir repo-path output-dir options)
+                export-css (if (fs/existsSync export-css-path) (str (fs/readFileSync export-css-path)) "")
+                _ (fs/writeFileSync (node-path/join output-static-dir "css" "export.css")  export-css)
+                custom-css (if (fs/existsSync custom-css-path) (str (fs/readFileSync custom-css-path)) "")
+                _ (fs/writeFileSync (node-path/join output-static-dir "css" "custom.css") custom-css)
+                _ (cleanup-js-dir output-static-dir)]
+               (notification-fn {:type "success"
+                                 :payload (str "Export public pages and publish assets to " output-dir " successfully 🎉")}))
+        (p/catch (fn [error]
+                   (notification-fn {:type "error"
+                                     :payload (str "Export public pages unexpectedly failed with: " error)}))))))

deps/publish-spa/src/logseq/publish_spa/html.cljs

@@ -3,6 +3,8 @@
             [goog.string :as gstring]
             [goog.string.format]))
 
+;; Copied from hiccup but tweaked for publish usage
+;; Any changes here should also be made in frontend.publishing/unescape-html
 (defn- escape-html
   "Change special characters into HTML character entities."
   [text]
@@ -24,16 +26,18 @@
       (gstring/format "<%s%s>%s</%s>\n" tag-name (html attrs) (html elts) tag-name))
     (map? v)
     (string/join ""
-              (map (fn [[k v]]
-                     (gstring/format " %s=\"%s\"" (name k) v)) v))
+                 (keep (fn [[k v]]
+                         ;; Skip nil values because some html tags haven't been
+                         ;; given values through html-options
+                         (when (some? v)
+                           (gstring/format " %s=\"%s\"" (name k) v))) v))
     (seq? v)
     (string/join " " (map html v))
     :else (str v)))
 
 (defn publishing-html
-  [transit-db app-state]
-  ;; TODO: Implement get-config
-  (let [{:keys [icon name alias title description url]} (:project {} #_(state/get-config))
+  [transit-db app-state options]
+  (let [{:keys [icon name alias title description url]} options
         icon (or icon "static/img/logo.png")
         project (or alias name)]
     (str "<!DOCTYPE html>\n"
@@ -83,7 +87,7 @@
            [:body
             [:div {:id "root"}]
             [:script (gstring/format "window.logseq_db=%s" (js/JSON.stringify (escape-html transit-db)))]
-            [:script (str "window.logseq_state=" (js/JSON.stringify app-state))]
+            [:script (str "window.logseq_state=" (js/JSON.stringify (pr-str app-state)))]
             [:script {:type "text/javascript"}
              "// Single Page Apps for GitHub Pages
       // https://github.com/rafgraph/spa-github-pages