diff --git a/.github/workflows/publishing.yml b/.github/workflows/publishing.yml new file mode 100644 index 00000000000..9847fdfdacb --- /dev/null +++ b/.github/workflows/publishing.yml @@ -0,0 +1,102 @@ +name: logseq/publishing CI + +on: + # Path filters ensure jobs only kick off if a change is made to publishing or + # its local dependencies + push: + branches: [master] + paths: + - 'deps/publishing/**' + # db is a local dep that could break functionality in this lib and should trigger this + - 'deps/db/**' + - '.github/workflows/publishing.yml' + - '!deps/publishing/**.md' + pull_request: + branches: [master] + paths: + - 'deps/publishing/**' + - 'deps/db/**' + - '.github/workflows/publishing.yml' + - '!deps/publishing/**.md' + +defaults: + run: + working-directory: deps/publishing + +env: + CLOJURE_VERSION: '1.10.1.763' + # This is the same as 1.8. + JAVA_VERSION: '11' + # This is the latest node version we can run. + NODE_VERSION: '18' + BABASHKA_VERSION: '1.0.168' + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Node + uses: actions/setup-node@v3 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'yarn' + cache-dependency-path: deps/publishing/yarn.lock + + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: ${{ env.JAVA_VERSION }} + + - name: Set up Babashka + uses: DeLaGuardo/setup-clojure@10.1 + with: + bb: ${{ env.BABASHKA_VERSION }} + + - name: Fetch yarn deps + run: yarn install --frozen-lockfile + + - name: Run nbb-logseq tests + run: yarn test + + # In this job because it depends on an npm package + - name: Load namespaces into nbb-logseq + run: bb test:load-all-namespaces-with-nbb . + + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: ${{ env.JAVA_VERSION }} + + - name: Set up Clojure + uses: DeLaGuardo/setup-clojure@10.1 + with: + cli: ${{ env.CLOJURE_VERSION }} + bb: ${{ env.BABASHKA_VERSION }} + + - name: Run clj-kondo lint + run: clojure -M:clj-kondo --lint src test + + - name: Carve lint for unused vars + run: bb lint:carve + + - name: Lint for vars that are too large + run: bb lint:large-vars + + - name: Lint for namespaces that aren't documented + run: bb lint:ns-docstrings + + - name: Lint for public vars that are private based on usage + run: bb lint:minimize-public-vars diff --git a/bb.edn b/bb.edn index ae45ab52ffe..30ec230e010 100644 --- a/bb.edn +++ b/bb.edn @@ -34,9 +34,9 @@ dev:build-publishing logseq.tasks.dev/build-publishing - dev:publish-spa + dev:publishing {:depends [dev:build-publishing] - :doc "Build publish spa app given graph and output dirs" + :doc "Build publishing spa app given graph and output dirs" :task (apply shell {:dir "scripts"} "yarn -s nbb-logseq -cp src -m logseq.tasks.dev.publishing" (into ["static"] *command-line-args*))} diff --git a/deps.edn b/deps.edn index e79e783f049..7a9b82c70f0 100644 --- a/deps.edn +++ b/deps.edn @@ -29,7 +29,7 @@ org.clojars.mmb90/cljs-cache {:mvn/version "0.1.4"} logseq/common {:local/root "deps/common"} logseq/graph-parser {:local/root "deps/graph-parser"} - logseq/publish-spa {:local/root "deps/publish-spa"} + logseq/publishing {:local/root "deps/publishing"} metosin/malli {:mvn/version "0.10.0"} fipp/fipp {:mvn/version "0.6.26"}} diff --git a/deps/db/README.md b/deps/db/README.md index 78f56025dc3..cc34e5f444d 100644 --- a/deps/db/README.md +++ b/deps/db/README.md @@ -13,7 +13,7 @@ two main namespaces, `logseq.db` and `logseq.db.rules`. ## Usage -See usage in `deps/graph-parser` and in the Logseq app. +See the frontend for example usage. ## Dev diff --git a/deps/graph-parser/README.md b/deps/graph-parser/README.md index 25d920e0032..791d4008a8c 100644 --- a/deps/graph-parser/README.md +++ b/deps/graph-parser/README.md @@ -18,7 +18,9 @@ CLIs. ## Usage -See `logseq.graph-parser.cli-test` for now. A real world example is coming soon. +See `logseq.graph-parser.cli-test` and [nbb-logseq example +scripts](https://github.com/logseq/nbb-logseq/tree/main/examples) for example +usage. ## Dev @@ -34,11 +36,11 @@ To run linters and tests, you'll want to install yarn dependencies once: yarn install ``` -This step is not needed if you're just running the application. +This step is not needed if you're just running the frontend application. ### Testing -Since this file is compatible with cljs and nbb-logseq, tests are run against both languages. +Since this library is compatible with cljs and nbb-logseq, tests are run against both languages. ClojureScript tests use https://github.com/Olical/cljs-test-runner. To run tests: ``` diff --git a/deps/publish-spa/deps.edn b/deps/publish-spa/deps.edn deleted file mode 100644 index 1fd650516a6..00000000000 --- a/deps/publish-spa/deps.edn +++ /dev/null @@ -1,3 +0,0 @@ -{:paths ["src"] - :deps - {logseq/db {:local/root "../db"}}} diff --git a/deps/publishing/.carve/config.edn b/deps/publishing/.carve/config.edn new file mode 100644 index 00000000000..f2c2dec805a --- /dev/null +++ b/deps/publishing/.carve/config.edn @@ -0,0 +1,4 @@ +{:paths ["src"] + :api-namespaces [;; for nbb-logseq CLIs + logseq.publishing] + :report {:format :ignore}} diff --git a/deps/publish-spa/.clj-kondo/config.edn b/deps/publishing/.clj-kondo/config.edn similarity index 81% rename from deps/publish-spa/.clj-kondo/config.edn rename to deps/publishing/.clj-kondo/config.edn index 4b701ff0340..24a43335cdd 100644 --- a/deps/publish-spa/.clj-kondo/config.edn +++ b/deps/publishing/.clj-kondo/config.edn @@ -7,7 +7,7 @@ {:aliases {clojure.string string}}} :lint-as - {logseq.publish-spa.test.helper/deftest-async clojure.test/deftest + {logseq.publishing.test.helper/deftest-async clojure.test/deftest promesa.core/let clojure.core/let} :skip-comments true :output {:progress true}} diff --git a/deps/publish-spa/.gitignore b/deps/publishing/.gitignore similarity index 100% rename from deps/publish-spa/.gitignore rename to deps/publishing/.gitignore diff --git a/deps/publishing/README.md b/deps/publishing/README.md new file mode 100644 index 00000000000..85b77fc9238 --- /dev/null +++ b/deps/publishing/README.md @@ -0,0 +1,52 @@ +## Description + +This library handles exporting the `frontend.publishing` single page +application. This library is compatible with ClojureScript and with +node/[nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide +frontend and Electron/commandline functionality. + +## API + +This library is under the parent namespace `logseq.publishing`. This library +provides two namespaces for node/CLI contexts, `logseq.publishing` and +`logseq.publishing.export` and two namespaces for the frontend, +`logseq.publishing.html` and `logseq.publishing.db`. + +## Usage + +See `logseq.tasks.dev.publishing` for a CLI example. See the frontend for cljs usage. + +## Dev + +This follows the practices that [the Logseq frontend +follows](/docs/dev-practices.md). Most of the same linters are used, with +configurations that are specific to this library. See [this library's CI +file](/.github/workflows/publishing.yml) for linting examples. + +### Setup + +To run linters and tests, you'll want to install yarn dependencies once: +``` +yarn install +``` + +This step is not needed if you're just running the frontend application. + +### Testing + +Testing is done with nbb-logseq and +[nbb-test-runner](https://github.com/nextjournal/nbb-test-runner). Some basic +usage: + +``` +# Run all tests +$ yarn test +# List available options +$ yarn test -H +# Run tests with :focus metadata flag +$ yarn test -i focus +``` + +### Managing dependencies + +See [standard nbb/cljs library advice in graph-parser](/deps/graph-parser/README.md#managing-dependencies). diff --git a/deps/publishing/bb.edn b/deps/publishing/bb.edn new file mode 100644 index 00000000000..234c14a0b4e --- /dev/null +++ b/deps/publishing/bb.edn @@ -0,0 +1,30 @@ +{:min-bb-version "1.0.168" + :deps + {logseq/bb-tasks + #_{:local/root "../../../bb-tasks"} + {:git/url "https://github.com/logseq/bb-tasks" + :git/sha "0d49051909bfa0c6b414e86606d82b4ea54f382c"}} + + :pods + {clj-kondo/clj-kondo {:version "2023.03.17"}} + + :tasks + {test:load-all-namespaces-with-nbb + logseq.bb-tasks.nbb.test/load-all-namespaces + + lint:large-vars + logseq.bb-tasks.lint.large-vars/-main + + lint:carve + logseq.bb-tasks.lint.carve/-main + + lint:ns-docstrings + logseq.bb-tasks.lint.ns-docstrings/-main + + lint:minimize-public-vars + logseq.bb-tasks.lint.minimize-public-vars/-main} + + :tasks/config + {:large-vars + ;; For vars with long running html + {:metadata-exceptions #{:large-vars/html}}}} diff --git a/deps/publishing/deps.edn b/deps/publishing/deps.edn new file mode 100644 index 00000000000..ca1078f3d2e --- /dev/null +++ b/deps/publishing/deps.edn @@ -0,0 +1,7 @@ +{:paths ["src"] + :deps + {logseq/db {:local/root "../db"}} + + :aliases + {:clj-kondo {:replace-deps {clj-kondo/clj-kondo {:mvn/version "2023.03.17"}} + :main-opts ["-m" "clj-kondo.main"]}}} diff --git a/deps/publish-spa/nbb.edn b/deps/publishing/nbb.edn similarity index 100% rename from deps/publish-spa/nbb.edn rename to deps/publishing/nbb.edn diff --git a/deps/publish-spa/package.json b/deps/publishing/package.json similarity index 88% rename from deps/publish-spa/package.json rename to deps/publishing/package.json index 3eea4948890..2e53d40dbf1 100644 --- a/deps/publish-spa/package.json +++ b/deps/publishing/package.json @@ -1,5 +1,5 @@ { - "name": "@logseq/publish-spa", + "name": "@logseq/publishing", "version": "1.0.0", "private": true, "devDependencies": { diff --git a/deps/publish-spa/src/logseq/publish_spa.cljs b/deps/publishing/src/logseq/publishing.cljs similarity index 86% rename from deps/publish-spa/src/logseq/publish_spa.cljs rename to deps/publishing/src/logseq/publishing.cljs index e16b09127ae..09f045c8522 100644 --- a/deps/publish-spa/src/logseq/publish_spa.cljs +++ b/deps/publishing/src/logseq/publishing.cljs @@ -1,7 +1,7 @@ -(ns logseq.publish-spa +(ns logseq.publishing "This node only ns provides api fns for exporting a publishing app" - (:require [logseq.publish-spa.html :as publish-html] - [logseq.publish-spa.export :as publish-export])) + (:require [logseq.publishing.html :as publish-html] + [logseq.publishing.export :as publish-export])) (defn export "Exports the given graph-dir and db to the specific output-dir. Most of the graph diff --git a/deps/publish-spa/src/logseq/publish_spa/db.cljs b/deps/publishing/src/logseq/publishing/db.cljs similarity index 97% rename from deps/publish-spa/src/logseq/publish_spa/db.cljs rename to deps/publishing/src/logseq/publishing/db.cljs index c9238fe9d65..c6087642d6a 100644 --- a/deps/publish-spa/src/logseq/publish_spa/db.cljs +++ b/deps/publishing/src/logseq/publishing/db.cljs @@ -1,9 +1,10 @@ -(ns logseq.publish-spa.db +(ns logseq.publishing.db + "Provides db fns and associated util fns for publishing" (:require [datascript.core :as d] [logseq.db.schema :as db-schema] [clojure.string :as string])) -(defn get-area-block-asset-url +(defn ^:api get-area-block-asset-url "Returns asset url for an area block used by pdf assets. This lives in this ns because it is used by this dep and needs to be indepent from the frontend app" [block page] diff --git a/deps/publish-spa/src/logseq/publish_spa/export.cljs b/deps/publishing/src/logseq/publishing/export.cljs similarity index 98% rename from deps/publish-spa/src/logseq/publish_spa/export.cljs rename to deps/publishing/src/logseq/publishing/export.cljs index f096c0f5053..f1241fb9dad 100644 --- a/deps/publish-spa/src/logseq/publish_spa/export.cljs +++ b/deps/publishing/src/logseq/publishing/export.cljs @@ -1,4 +1,4 @@ -(ns logseq.publish-spa.export +(ns logseq.publishing.export "This electron only ns (for the main process) exports files from multiple locations to provide a complete publishing app" (:require ["fs-extra" :as fse] @@ -6,11 +6,11 @@ ["fs" :as fs] [promesa.core :as p])) -(def js-files +(def ^:api js-files "js files from publishing release build" ["main.js" "code-editor.js" "excalidraw.js" "tldraw.js"]) -(def static-dirs +(def ^:api static-dirs "dirs under static dir to copy over" ["css" "fonts" "icons" "img" "js"]) diff --git a/deps/publish-spa/src/logseq/publish_spa/html.cljs b/deps/publishing/src/logseq/publishing/html.cljs similarity index 97% rename from deps/publish-spa/src/logseq/publish_spa/html.cljs rename to deps/publishing/src/logseq/publishing/html.cljs index 3f2d51ab3f5..db8fd334c5c 100644 --- a/deps/publish-spa/src/logseq/publish_spa/html.cljs +++ b/deps/publishing/src/logseq/publishing/html.cljs @@ -1,11 +1,11 @@ -(ns logseq.publish-spa.html +(ns logseq.publishing.html "This frontend only ns builds the publishing html including doing all the necessary db filtering" (:require [clojure.string :as string] [goog.string :as gstring] [goog.string.format] [datascript.transit :as dt] - [logseq.publish-spa.db :as db])) + [logseq.publishing.db :as db])) ;; Copied from hiccup but tweaked for publish usage ;; Any changes here should also be made in frontend.publishing/unescape-html @@ -20,7 +20,7 @@ necessary db filtering" (string/replace "'" "logseq____'"))) ;; Copied from https://github.com/babashka/babashka/blob/8c1077af00c818ade9e646dfe1297bbe24b17f4d/examples/notes.clj#L21 -(defn html [v] +(defn- html [v] (cond (vector? v) (let [tag (first v) attrs (second v) @@ -39,7 +39,7 @@ necessary db filtering" (string/join " " (map html v)) :else (str v))) -(defn publishing-html +(defn- ^:large-vars/html publishing-html [transit-db app-state options] (let [{:keys [icon name alias title description url]} options icon (or icon "static/img/logo.png") diff --git a/deps/publish-spa/test/logseq/publish_spa/db_test.cljs b/deps/publishing/test/logseq/publishing/db_test.cljs similarity index 97% rename from deps/publish-spa/test/logseq/publish_spa/db_test.cljs rename to deps/publishing/test/logseq/publishing/db_test.cljs index 6e40c8bdec0..e47c72ed255 100644 --- a/deps/publish-spa/test/logseq/publish_spa/db_test.cljs +++ b/deps/publishing/test/logseq/publishing/db_test.cljs @@ -1,7 +1,7 @@ -(ns logseq.publish-spa.db-test +(ns logseq.publishing.db-test (:require [cljs.test :refer [deftest is]] [clojure.set :as set] - [logseq.publish-spa.db :as publish-db] + [logseq.publishing.db :as publish-db] [logseq.graph-parser :as graph-parser] [datascript.core :as d] [logseq.db :as ldb])) diff --git a/deps/publish-spa/test/logseq/publish_spa/export_test.cljs b/deps/publishing/test/logseq/publishing/export_test.cljs similarity index 95% rename from deps/publish-spa/test/logseq/publish_spa/export_test.cljs rename to deps/publishing/test/logseq/publishing/export_test.cljs index 163f08b5551..4ea36143b21 100644 --- a/deps/publish-spa/test/logseq/publish_spa/export_test.cljs +++ b/deps/publishing/test/logseq/publishing/export_test.cljs @@ -1,7 +1,7 @@ -(ns logseq.publish-spa.export-test +(ns logseq.publishing.export-test (:require [cljs.test :as t :refer [is use-fixtures async]] - [logseq.publish-spa.test.helper :as test-helper :include-macros true :refer [deftest-async]] - [logseq.publish-spa.export :as publish-export] + [logseq.publishing.test.helper :as test-helper :include-macros true :refer [deftest-async]] + [logseq.publishing.export :as publish-export] [promesa.core :as p] [clojure.set :as set] ["fs" :as fs] diff --git a/deps/publish-spa/test/logseq/publish_spa/test/helper.clj b/deps/publishing/test/logseq/publishing/test/helper.clj similarity index 95% rename from deps/publish-spa/test/logseq/publish_spa/test/helper.clj rename to deps/publishing/test/logseq/publishing/test/helper.clj index 67ddd80df5a..8607d386be2 100644 --- a/deps/publish-spa/test/logseq/publish_spa/test/helper.clj +++ b/deps/publishing/test/logseq/publishing/test/helper.clj @@ -1,4 +1,4 @@ -(ns logseq.publish-spa.test.helper) +(ns logseq.publishing.test.helper) ;; Copied from https://github.com/babashka/nbb/blob/e5d84b0fac59774f5d7a4a9e807240cce04bf252/test/nbb/test_macros.clj (defmacro deftest-async diff --git a/deps/publish-spa/yarn.lock b/deps/publishing/yarn.lock similarity index 100% rename from deps/publish-spa/yarn.lock rename to deps/publishing/yarn.lock diff --git a/docs/dev-practices.md b/docs/dev-practices.md index 24fc93af6dc..4914ec40c8b 100644 --- a/docs/dev-practices.md +++ b/docs/dev-practices.md @@ -257,10 +257,29 @@ We strive to use explicit names that are self explanatory so that our codebase i ### Babashka tasks -There are a number of bb tasks under `dev:` for developers. There are also some -tasks under `nbb:` which are useful for inspecting database changes in realtime. -See [these docs](https://github.com/logseq/bb-tasks#logseqbb-tasksnbbwatch) for -more info. +There are a number of bb tasks under `dev:` for developers. Some useful ones to +point out: + +* `dev:validate-repo-config-edn` - Validate a repo config.edn + + ```sh + bb dev:validate-repo-config-edn templates/config.edn + ``` + +* `dev:publishing` - Build a publishing app for a given graph dir. If the + publishing frontend is out of date, it builds that first which takes time. + Subsequent runs are quick. + + ```sh + # Build the export + $ bb dev:publishing /path/to/graph-dir tmp/publish + # View the app in a browser + $ open tmp/publish/index.html + ``` + +There are also some tasks under `nbb:` which are useful for inspecting database +changes in realtime. See [these +docs](https://github.com/logseq/bb-tasks#logseqbb-tasksnbbwatch) for more info. ### Dev Commands diff --git a/scripts/nbb.edn b/scripts/nbb.edn index 69564092a10..7076f055dc4 100644 --- a/scripts/nbb.edn +++ b/scripts/nbb.edn @@ -3,5 +3,5 @@ {logseq/graph-parser ;; Nbb bug. Should just be "../graph-parser" {:local/root "../../../../deps/graph-parser"} - logseq/publish-spa - {:local/root "../../../../deps/publish-spa"}}} + logseq/publishing + {:local/root "../../../../deps/publishing"}}} diff --git a/scripts/src/logseq/tasks/dev/publishing.cljs b/scripts/src/logseq/tasks/dev/publishing.cljs index 6bfc4176117..1853f934348 100644 --- a/scripts/src/logseq/tasks/dev/publishing.cljs +++ b/scripts/src/logseq/tasks/dev/publishing.cljs @@ -1,6 +1,7 @@ (ns logseq.tasks.dev.publishing + "Basic script for publishing from CLI" (:require [logseq.graph-parser.cli :as gp-cli] - [logseq.publish-spa :as publish-spa] + [logseq.publishing :as publishing] ["fs" :as fs] ["path" :as node-path] [clojure.edn :as edn])) @@ -18,8 +19,8 @@ ;; Offset relative paths since they are run in a different directory than user is in (map #(if (node-path/isAbsolute %) % (node-path/resolve ".." %)) args) repo-config (-> (node-path/join graph-dir "logseq" "config.edn") fs/readFileSync str edn/read-string)] - (publish-spa/export (get-db graph-dir) - static-dir - graph-dir - output-path - {:repo-config repo-config}))) + (publishing/export (get-db graph-dir) + static-dir + graph-dir + output-path + {:repo-config repo-config}))) diff --git a/src/electron/electron/core.cljs b/src/electron/electron/core.cljs index 83d75ff02d3..637546a4c56 100644 --- a/src/electron/electron/core.cljs +++ b/src/electron/electron/core.cljs @@ -22,7 +22,7 @@ [electron.window :as win] [electron.exceptions :as exceptions] ["/electron/utils" :as js-utils] - [logseq.publish-spa.export :as publish-export])) + [logseq.publishing.export :as publish-export])) ;; Keep same as main/frontend.util.url (defonce LSP_SCHEME "logseq") diff --git a/src/main/frontend/extensions/pdf/utils.cljs b/src/main/frontend/extensions/pdf/utils.cljs index 6b9c292f6c9..969a021fbe2 100644 --- a/src/main/frontend/extensions/pdf/utils.cljs +++ b/src/main/frontend/extensions/pdf/utils.cljs @@ -4,7 +4,7 @@ [frontend.util :as util] ["/frontend/extensions/pdf/utils" :as js-utils] [datascript.core :as d] - [logseq.publish-spa.db :as publish-db] + [logseq.publishing.db :as publish-db] [clojure.string :as string])) (defonce MAX-SCALE 5.0) diff --git a/src/main/frontend/handler/export.cljs b/src/main/frontend/handler/export.cljs index 3ef20af171e..6b5f8fb29d4 100644 --- a/src/main/frontend/handler/export.cljs +++ b/src/main/frontend/handler/export.cljs @@ -15,7 +15,7 @@ [frontend.mobile.util :as mobile-util] [frontend.modules.file.core :as outliner-file] [frontend.modules.outliner.tree :as outliner-tree] - [logseq.publish-spa.html :as publish-html] + [logseq.publishing.html :as publish-html] [frontend.state :as state] [frontend.util :as util] [frontend.util.property :as property]