Enhance: Introduce global address space and HTTP client/server. (#639)

* Make connection watchable, readable and track store-id.

* Fix format.

* Fix print tests.

* Fix pod test.

* Derandomize store-id.

* Needs negation.

* Fix test strings.

* Print commit-id. Add readers for history dbs.

* Add readers.cljc.

* Try to make commit-ids deterministic.

* Try to replace commit-ids in print tests.

* Syntax error.

* Remove commit-id from strings in test.

* Replace commit-id instead to test commit-id as well.

* Unify API spec. Re-implement http server and writer and add client.

* Fix format and http tests.

* Add http deps to test profile.

* Syntax error.

* Fix load-entities.

* Fix remote-peer handling.

* Add http dependencies to native build.

* Fix reflection problems, add server to CLI.

* Add sandbox for client.

* Add babashka http client by default.

* Fix format.

* Remove print statements.

* Remove redundant connection handling in datahike.core.

* Remove from versioning as well.

* Properly call listen callbacks.

* Pass writer through.

* Do not use regex because of CI/CD mismatch.

* Fix second test.

* Format fix.

* Remove print statement.

* Update GraalVM.

* Remove debug prn.

* Remove server from CLI.

* Update deps, fix CLI.

* deps.edn

* Factor out http-server into separate directory.

* deps.edn

* Fix entity equality and speed up DB equality.

* update build tooling

* Add server uber jar build.

* Expose CORS config options, don't log token. Minor fixes.

* Remove println.

* Add scope to memory store.

* Set memory scope in tests. Fix format.

* Fix config test.

* Fix pod test.

* Fix bb pod test.

* Use gensym for local bindings in client.clj, remove stale arguments.

* Does not work for fn binding.

* Fix print-method preference for remote records.

* Quote def manually.

* Fix client.

* Incorporate feedback.

* Use GET requests for pure functions to facilitate HTTP caching.

* Support simple GET caching.

* Connection management should not be pure.

* Be precise about memory semantics. Fix db.

* Fix database-exists?, remove tempid from remote API.

* Extract remote-peer hygienically.

* Remove tempid from server test.

* Implement base case for remote-peer.

* Add documentation.

* Fix native test, polish docs.

* Complete command line renaming.

* Update CircleCI. Add sharding note.

* Add AWS lambda section.

* Typo.

* Add JSON support to server and client.

* Add jsonista to native profiles.

* Add datahike.json.

* Move jsonista to main deps.

* Fix swagger sets, add JSON docs, prefix ! to tagged literals.

* Add network diagram.

* Fix path.

* Set white background color.

* Set white background box.

* Use JavaScript syntax highlighting.

* Improve wording.

* Add distributed index space.

* Improve diagram.

* Annotate HTTP request type.

* Add ASCII logo to server and cli.

* Log config and version regularly.

* Fix env variables.

* Test: replace cheshire with jsonista.

* Bump GraalVM dependency.

* Fix GraalVM paths.

* gu command was removed, native-image should be around.

* Ensure to use shared distributed store in example.

* Remove duplicate entry from deps.

* Fix typos.

* feat: adding exception middleware (#645)

* Remove unused middleware.

---------

Co-authored-by: Timo Kramer <fw-github@timokramer.de>
Co-authored-by: Timo Kramer <4785848+TimoKramer@users.noreply.github.com>

.circleci/config.yml

@@ -75,11 +75,10 @@ jobs:
           name: install graalvm
           command: |
             cd /home/circleci
-            wget https://github.com/graalvm/graalvm-ce-builds/releases/download/vm-22.2.0/graalvm-ce-java17-linux-amd64-22.2.0.tar.gz
-            tar -xzf graalvm-ce-java17-linux-amd64-22.2.0.tar.gz
-            export PATH=/home/circleci/graalvm-ce-java17-22.2.0/bin:$PATH
-            export JAVA_HOME=/home/circleci/graalvm-ce-java17-22.2.0/
-            gu install native-image
+            wget https://github.com/graalvm/graalvm-ce-builds/releases/download/jdk-21.0.0/graalvm-community-jdk-21.0.0_linux-x64_bin.tar.gz
+            tar -xzf graalvm-community-jdk-21.0.0_linux-x64_bin.tar.gz
+            export PATH=/home/circleci/graalvm-community-openjdk-21+35.1/bin:$PATH
+            export JAVA_HOME=/home/circleci/graalvm-community-openjdk-21+35.1
       - run:
           name: install clojure
           command: |
@@ -99,13 +98,13 @@ jobs:
           name: build native-image
           command: |
             cd /home/circleci/replikativ
-            export PATH=/home/circleci/graalvm-ce-java17-22.2.0/bin:/home/circleci/clojure/bin:$PATH
-            export JAVA_HOME=/home/circleci/graalvm-ce-java17-22.2.0/
+            export PATH=/home/circleci/graalvm-community-openjdk-21+35.1/bin:/home/circleci/clojure/bin:$PATH
+            export JAVA_HOME=/home/circleci/graalvm-community-openjdk-21+35.1
             bb ni-cli
       - persist_to_workspace:
           root: /home/circleci/
           paths:
-            - replikativ/dhi
+            - replikativ/dthk
   persistent-set-test:
     executor: tools/clojurecli
     steps:
@@ -231,8 +230,8 @@ jobs:
           name: Release cli on GitHub
           command: bb release native-cli
           environment:
-            DHI_PLATFORM: linux
-            DHI_ARCH: x86_64
+            DTHK_PLATFORM: linux
+            DTHK_ARCH: x86_64
 
 workflows:
   build-test-and-deploy:

.clj-kondo/config.edn

@@ -3,7 +3,9 @@
            datahike.lru/defcache clojure.core/deftype
            datalog.parser.type/deftrecord clojure.core/defrecord}
  :linters {;; For some reason not detected as resolved
-           :unresolved-var {:exclude [babashka.fs]}
+           :unresolved-var {:exclude [babashka.fs
+                                      datahike.http.client
+                                      datahike.api]}
            ;; Used by bb.edn, but not detected as used 
            :clojure-lsp/unused-public-var {:exclude [tools.build/clean
                                                      tools.build/compile-clojure

.dir-locals.el

@@ -1,2 +1,2 @@
 ((clojure-mode . ((cider-preferred-build-tool . clojure-cli)
-                  (cider-clojure-cli-aliases . ":dev:test:libdatahike"))))
+                  (cider-clojure-cli-aliases . ":dev:test:libdatahike:http"))))

.gitignore

@@ -1,5 +1,7 @@
 /target
 /bench/target
+/libdatahike/target
+/target-http-server
 /classes
 /checkouts
 pom.xml.asc
@@ -29,7 +31,6 @@ out/
 .cache
 /datahike
 tests.user.edn
-/libdatahike/target
 libdatahike.zip
 dhi
 dhi.build_artifacts.txt

bb.edn

@@ -1,7 +1,7 @@
 {:min-bb-version "0.8.0"
  :pods {;since babashka 0.8.0, can only be declared in bb.edn
-        org.babashka/tools-deps-native {:version "0.0.5"}
-        clj-kondo/clj-kondo {:version "2022.10.05"}}
+        org.babashka/tools-deps-native {:version "0.1.1"}
+        clj-kondo/clj-kondo {:version "2023.07.13"}}
  :deps {datahike/bb {:local/root "bb"}}
  :tasks {:requires [[babashka.fs :as fs]
                     [clojure.edn :as edn]
@@ -80,6 +80,35 @@
          release {:doc "Build and release jar to GitHub"
                   :task (release/-main config *command-line-args*)}
 
+
+         ;; http server build and release
+
+         http-server-clean {:doc  "Remove build files"
+                            :task (build/clean (-> config :build :http-server-clj))}
+
+         http-server-jcompile {:doc     "Compile java classes"
+                               :depends [http-server-clean]
+                               :task    (build/compile-java (-> config :build :http-server-clj))}
+
+         http-server-ccompile {:doc  "Compile clojure namespaces"
+                               :task (build/compile-clojure (-> config :build :http-server-clj))}
+
+         http-server-pom {:doc  "Create pom file"
+                          :task (build/pom config (-> config :build :http-server-clj))}
+
+         http-server-uber {:doc     "Build jar"
+                           :depends [http-server-jcompile http-server-pom http-server-ccompile]
+                           :task    (build/uber config (-> config :build :http-server-clj))}
+
+         http-server-install {:doc  "Install uber jar locally"
+                              :task (deploy/local config (-> config :build :http-server-clj))}
+
+         http-server-release {:doc     "Build and release jar to GitHub"
+                              :depends [http-server-uber]
+                              :task    (let [jar (build/jar-path config (-> config :build :http-server-clj))]
+                                         (release/gh-release jar config))}
+
+
          ;; native image
 
          ni-check {:doc "Check for 'native-image' program"

bb/deps.edn

@@ -2,7 +2,7 @@
  :deps {org.babashka/spec.alpha {:git/url "https://github.com/babashka/spec.alpha"
                                  :git/sha "644a7fc216e43d5da87b07471b0f87d874107d1a"}
         ;; https://github.com/babashka/tools.bbuild
-        io.github.babashka/tools.bbuild {:git/sha "0a4959d2b1147f79e9fb37e0243727390b8ba2b3"}
+        io.github.babashka/tools.bbuild {:git/sha "73e4d0a26c65cdf1d4c7bf7e9e46e6e5f1978a37"}
         ;; https://github.com/borkdude/gh-release-artifact
         io.github.borkdude/gh-release-artifact {:git/sha "05f8d8659e6805d513c59447ff41dc8497878462"}
         selmer/selmer {:mvn/version "1.12.55"}}}

bb/resources/native-image-tests/run-bb-pod-tests.clj

@@ -11,7 +11,7 @@
 (def config {:keep-history? true,
              :search-cache-size 10000,
              :index :datahike.index/persistent-set,
-             :store {:id "inexpensive-red-fox", :backend :mem},
+             :store {:id "inexpensive-red-fox", :backend :mem :scope "test.datahike.io"},
              :store-cache-size 1000,
              :attribute-refs? false,
              :writer {:backend :self},
@@ -22,14 +22,14 @@
 (deftest pod-workflow
 
   (testing "delete-database"
-    (is (= {}
+    (is (= nil
            (d/delete-database config))))
 
   (testing "create-database"
     (is (= {:keep-history? true
             :search-cache-size 10000
             :index :datahike.index/persistent-set
-            :store {:id "inexpensive-red-fox", :backend :mem}
+            :store {:id "inexpensive-red-fox", :backend :mem :scope "test.datahike.io"}
             :store-cache-size 1000
             :attribute-refs? false
             :writer {:backend :self}
@@ -44,7 +44,7 @@
 
   (let [conn (d/connect config)]
     (testing "connect"
-      (is (= "conn:-1033483489"
+      (is (= "conn:2061701040"
              conn)))
     (testing "transact"
       (is (= [:tempids :db-before :db-after :tx-meta :tx-data]
@@ -55,11 +55,11 @@
     (testing "transact with bad arg"
       (is (thrown? clojure.lang.ExceptionInfo
                    (keys (d/transact
-                           "foo"
-                           [{:name  "Alice", :age   20}
-                            {:name  "Bob", :age   30}
-                            {:name  "Charlie", :age   40}
-                            {:age 15}])))))
+                          "foo"
+                          [{:name  "Alice", :age   20}
+                           {:name  "Bob", :age   30}
+                           {:name  "Charlie", :age   40}
+                           {:age 15}])))))
     (testing "with-db"
       (is (= #{[2 "Bob" 30] [1 "Alice" 20] [3 "Charlie" 40]}
              (d/with-db [db (d/db conn)]
@@ -78,12 +78,12 @@
                             :where
                             [[?e :name ?n]
                              [?e :age ?a]]}
-                          :args [(d/db conn)]})
+                   :args [(d/db conn)]})
              (d/q '[:find ?e ?n ?a
                     :where
                     [?e :name ?n]
                     [?e :age ?a]]
-                (d/db conn)))))
+                  (d/db conn)))))
     (let [timestamp (Date.)]
       (d/transact conn {:tx-data [{:db/id 3 :age 25}]})
       (d/transact conn [{:name "FOO"  :age "BAR"}])
@@ -117,17 +117,17 @@
       (testing "since tx-id"
         (is (= #{[5 "FOO" "BAR"]}
                (d/q '[:find ?e ?n ?a
-                       :where
-                       [?e :name ?n]
-                       [?e :age ?a]]
+                      :where
+                      [?e :name ?n]
+                      [?e :age ?a]]
                     (d/since (d/db conn) 536870914)))))
       (testing "since timestamp"
         (is (= #{[5 "FOO" "BAR"]}
                (d/q '[:find ?e ?n ?a
-                       :where
-                       [?e :name ?n]
+                      :where
+                      [?e :name ?n]
                       [?e :age ?a]]
-                     (d/since (d/db conn) timestamp)))))
+                    (d/since (d/db conn) timestamp)))))
       (testing "history"
         (is (= #{[3 "Charlie" 25] [2 "Bob" 30] [5 "FOO" "BAR"] [1 "Alice" 20] [3 "Charlie" 40]}
                (d/q '[:find ?e ?n ?a

bb/resources/native-image-tests/run-native-image-tests

@@ -8,14 +8,14 @@ CONFIG=bb/resources/native-image-tests/testconfig.edn
 
 trap "rm -rf $TMPSTORE" EXIT
 
-./dhi delete-database $CONFIG
-./dhi create-database $CONFIG
+./dthk delete-database $CONFIG
+./dthk create-database $CONFIG
 
-./dhi database-exists $CONFIG
+./dthk database-exists $CONFIG
 
-./dhi benchmark conn:$CONFIG 0 100000 10000
-./dhi transact conn:$CONFIG '[[:db/add -1 :name "Judea"]]'
-QUERY_OUT=`./dhi query '[:find (count ?e) . :where [?e :name _]]' db:$CONFIG`
+./dthk benchmark conn:$CONFIG 0 100000 10000
+./dthk transact conn:$CONFIG '[[:db/add -1 :name "Judea"]]'
+QUERY_OUT=`./dthk query '[:find (count ?e) . :where [?e :name _]]' db:$CONFIG`
 
 if [ $QUERY_OUT -eq 100001 ]
 then
@@ -26,22 +26,22 @@ else
 fi
 
 # test history input parsing
-./dhi query '[:find (count ?e) . :where [?e :name _]]' history:$CONFIG
-./dhi query '[:find (count ?e) . :where [?e :name _]]' since:0:$CONFIG
-./dhi query '[:find (count ?e) . :where [?e :name _]]' asof:0:$CONFIG
+./dthk query '[:find (count ?e) . :where [?e :name _]]' history:$CONFIG
+./dthk query '[:find (count ?e) . :where [?e :name _]]' since:0:$CONFIG
+./dthk query '[:find (count ?e) . :where [?e :name _]]' asof:0:$CONFIG
 
 
 # other calls
-./dhi pull db:$CONFIG "[:db/id, :name]" "1"
-./dhi pull-many db:$CONFIG "[:db/id, :name]" "[1]"
-./dhi entity db:$CONFIG "1"
-./dhi datoms db:$CONFIG "{:index :eavt :components [1]}"
-./dhi schema db:$CONFIG
-./dhi reverse-schema db:$CONFIG
-./dhi metrics db:$CONFIG
+./dthk pull db:$CONFIG "[:db/id, :name]" "1"
+./dthk pull-many db:$CONFIG "[:db/id, :name]" "[1]"
+./dthk entity db:$CONFIG "1"
+./dthk datoms db:$CONFIG "{:index :eavt :components [1]}"
+./dthk schema db:$CONFIG
+./dthk reverse-schema db:$CONFIG
+./dthk metrics db:$CONFIG
 
 # test serialization
-./dhi query '[:find ?e . :where [?e :name ?n]]' db:$CONFIG --format cbor >> /tmp/test
-./dhi query '[:find ?i :in $ ?i . :where [?e :name ?n]]' db:$CONFIG cbor:/tmp/test # => 1
+./dthk query '[:find ?e . :where [?e :name ?n]]' db:$CONFIG --format cbor >> /tmp/test
+./dthk query '[:find ?i :in $ ?i . :where [?e :name ?n]]' db:$CONFIG cbor:/tmp/test # => 1
 
-./dhi delete-database $CONFIG
+./dthk delete-database $CONFIG

bb/src/tools/build.clj

@@ -6,15 +6,14 @@
             [selmer.parser :refer [render]]
             [tools.version :as version :refer [read-edn-file]]))
 
-
 (defn clean [{:keys [target-dir] :as _project-config}]
   (print (str "Cleaning up target directory '" target-dir "'..."))
   (fs/delete-tree target-dir)
   (println "Done."))
 
 (defn basis [{:keys [deps-file aliases] :as _project-config}]
   (b/create-basis (cond-> {:project deps-file}
-                          aliases (assoc :aliases aliases))))
+                    aliases (assoc :aliases aliases))))
 
 (defn compile-java
   ([] (compile-java (read-edn-file "config.edn")))

bb/src/tools/release.clj

@@ -50,10 +50,10 @@
       (println (:url ret)))))
 
 (defn zip-path [lib version target-dir zip-pattern]
-  (let [platform (System/getenv "DHI_PLATFORM")
-        arch (System/getenv "DHI_ARCH")]
+  (let [platform (System/getenv "DTHK_PLATFORM")
+        arch (System/getenv "DTHK_ARCH")]
     (if-not (and platform arch)
-      (do (println "ERROR: Environment variables DHI_PLATFORM and DHI_ARCH need to be set")
+      (do (println "ERROR: Environment variables DTHK_PLATFORM and DTHK_ARCH need to be set")
           (System/exit 1))
       (str target-dir "/" (render zip-pattern {:platform platform
                                                :arch arch

bb/src/tools/test.clj

@@ -45,12 +45,12 @@
          "-X" "backward-test/read")))
 
 (defn native-image []
-  (if (fs/exists? "./dhi")
+  (if (fs/exists? "./dthk")
     (p/shell "./bb/resources/native-image-tests/run-native-image-tests")
     (println "Native image cli missing. Please run 'bb ni-cli' and try again.")))
 
 (defn bb-pod []
-  (if (fs/exists? "./dhi")
+  (if (fs/exists? "./dthk")
     (p/shell "./bb/resources/native-image-tests/run-bb-pod-tests.clj")
     (println "Native image cli missing. Please run 'bb ni-cli' and try again.")))
 

config.edn

@@ -17,7 +17,16 @@
                :deps-file "deps.edn"
                :jar-pattern "{{repo.lib}}-{{version-str}}.jar"
                :lib io.replikativ/datahike}
-
+         :http-server-clj {;; jvm build
+                           :src-dirs      ["src" "http-server"]
+                           :java-src-dirs ["java"]
+                           :target-dir    "target-http-server"
+                           :class-dir     "target-http-server/classes"
+                           :deps-file     "deps.edn"
+                           :jar-pattern   "{{repo.lib}}-http-server-{{version-str}}.jar"
+                           :aliases       [:http-server]
+                           :main          datahike.http.server
+                           :lib           io.replikativ/datahike-http-server}
          :native {;; jvm build
                   :src-dirs ["src" "libdatahike/src"]
                   :java-src-dirs ["java"]
@@ -35,5 +44,5 @@
                   :java-interface "libdatahike/src/datahike/impl/LibDatahike.java"}}
 
  :release {:native-cli {:target-dir "."
-                        :binary-name "dhi"
+                        :binary-name "dthk"
                         :zip-pattern "{{lib}}-{{version}}-{{platform}}-{{arch}}.zip"}}}