Feature/sorted sets

deps.edn

@@ -1,49 +1,48 @@
-{:deps  {org.clojure/clojure             {:mvn/version "1.11.1"}
-         org.clojure/clojurescript       {:mvn/version "1.11.60"}
-         org.clojure/core.async          {:mvn/version "1.6.673"}
-         org.clojure/core.cache          {:mvn/version "1.0.225"}
-         org.clojars.mmb90/cljs-cache    {:mvn/version "0.1.4"}
-         org.clojure/data.avl            {:mvn/version "0.1.0"}
-         org.clojure/data.xml            {:mvn/version "0.2.0-alpha8"}
-         environ/environ                 {:mvn/version "1.2.0"}
-         byte-streams/byte-streams       {:mvn/version "0.2.4"}
-         cheshire/cheshire               {:mvn/version "5.11.0"}
-         instaparse/instaparse           {:mvn/version "1.4.12"}
-         metosin/malli                   {:mvn/version "0.11.0"}
-         com.fluree/json-ld              {:git/url "https://github.com/fluree/json-ld.git"
-                                          :git/sha "2be635f2a30b08a5c100a35dd8afb20078d261ac"}
-
-
-         ;; logging
-         org.clojure/tools.logging       {:mvn/version "1.2.4"}
-         ch.qos.logback/logback-classic  {:mvn/version "1.4.7"}
-         org.slf4j/slf4j-api             {:mvn/version "2.0.7"}
-
-         ;; Lucene
-         clucie/clucie                   {:mvn/version "0.4.2"}
-
-         ;; http
-         http-kit/http-kit               {:mvn/version "2.6.0"}
-         com.fluree/http.async.client    {:mvn/version "1.3.1-25-0xae4f"}
-
-         ;; benchmarking
-         criterium/criterium             {:mvn/version "0.4.6"}
-
-         ;; serialization / compression
-         com.fluree/alphabase            {:mvn/version "3.3.0"}
-
-         ;; cryptography
-         com.fluree/crypto               {:mvn/version "0.4.0"}
-
-         org.bouncycastle/bcprov-jdk15on {:mvn/version "1.70"}
-
-         ;; smartfunctions
-         org.babashka/sci                {:mvn/version "0.3.31"}
-
-         ;; storage
-         com.cognitect.aws/api           {:mvn/version "0.8.666"}
-         com.cognitect.aws/endpoints     {:mvn/version "1.1.12.456"}
-         com.cognitect.aws/s3            {:mvn/version "847.2.1365.0"}}
+{:deps {org.clojure/clojure                         {:mvn/version "1.11.1"}
+        org.clojure/clojurescript                   {:mvn/version "1.11.60"}
+        org.clojure/core.async                      {:mvn/version "1.6.673"}
+        org.clojure/core.cache                      {:mvn/version "1.0.225"}
+        org.clojars.mmb90/cljs-cache                {:mvn/version "0.1.4"}
+        org.clojure/data.xml                        {:mvn/version "0.2.0-alpha8"}
+        environ/environ                             {:mvn/version "1.2.0"}
+        byte-streams/byte-streams                   {:mvn/version "0.2.4"}
+        cheshire/cheshire                           {:mvn/version "5.11.0"}
+        instaparse/instaparse                       {:mvn/version "1.4.12"}
+        metosin/malli                               {:mvn/version "0.11.0"}
+        com.fluree/json-ld                          {:git/url "https://github.com/fluree/json-ld.git"
+                                                     :git/sha "2be635f2a30b08a5c100a35dd8afb20078d261ac"}
+        persistent-sorted-set/persistent-sorted-set {:mvn/version "0.2.3"}
+
+
+        ;; logging
+        org.clojure/tools.logging      {:mvn/version "1.2.4"}
+        ch.qos.logback/logback-classic {:mvn/version "1.4.7"}
+        org.slf4j/slf4j-api            {:mvn/version "2.0.7"}
+
+        ;; Lucene
+        clucie/clucie {:mvn/version "0.4.2"}
+
+        ;; http
+        http-kit/http-kit            {:mvn/version "2.6.0"}
+        com.fluree/http.async.client {:mvn/version "1.3.1-25-0xae4f"}
+
+        ;; benchmarking
+        criterium/criterium {:mvn/version "0.4.6"}
+
+        ;; serialization / compression
+        com.fluree/alphabase {:mvn/version "3.3.0"}
+
+        ;; cryptography
+        com.fluree/crypto               {:mvn/version "0.4.0"}
+        org.bouncycastle/bcprov-jdk15on {:mvn/version "1.70"}
+
+        ;; smartfunctions
+        org.babashka/sci {:mvn/version "0.3.31"}
+
+        ;; storage
+        com.cognitect.aws/api       {:mvn/version "0.8.666"}
+        com.cognitect.aws/endpoints {:mvn/version "1.1.12.456"}
+        com.cognitect.aws/s3        {:mvn/version "847.2.1365.0"}}
 
  :paths ["src" "resources"]
 
@@ -63,8 +62,8 @@
 
   :cljtest
   {:extra-paths ["test" "dev-resources" "test-resources"]
-   :extra-deps  {lambdaisland/kaocha         {:mvn/version "1.83.1314"}
-                 org.clojure/test.check      {:mvn/version "1.1.1"}
+   :extra-deps  {lambdaisland/kaocha                   {:mvn/version "1.83.1314"}
+                 org.clojure/test.check                {:mvn/version "1.1.1"}
                  io.github.cap10morgan/test-with-files {:git/tag "v1.0.0"
                                                         :git/sha "9181a2e"}}
    :exec-fn     kaocha.runner/exec-fn
@@ -96,15 +95,15 @@
    :main-opts   ["-m" "cloverage.coverage" "-p" "src" "-s" "test" "--output" "scanning_results/coverage"]}
 
   :eastwood
-  {:extra-deps  {jonase/eastwood {:mvn/version "1.4.0"}}
-   :main-opts   ["-m" "eastwood.lint"
-                 {:source-paths ["src" "src-docs"]
-                  :test-paths   ["test"]
-                  ;; TODO: Un-exclude this when it stops triggering false
-                  ;;       positives on "UnsupportedOperationException empty is
-                  ;;       not supported on Flake" when using the #Flake data
-                  ;;       reader - WSM 2023-02-01
-                  :exclude-linters [:implicit-dependencies]}]}
+  {:extra-deps {jonase/eastwood {:mvn/version "1.4.0"}}
+   :main-opts  ["-m" "eastwood.lint"
+                {:source-paths    ["src" "src-docs"]
+                 :test-paths      ["test"]
+                 ;; TODO: Un-exclude this when it stops triggering false
+                 ;;       positives on "UnsupportedOperationException empty is
+                 ;;       not supported on Flake" when using the #Flake data
+                 ;;       reader - WSM 2023-02-01
+                 :exclude-linters [:implicit-dependencies]}]}
 
   :ancient
   {:extra-deps {com.github.liquidz/antq {:mvn/version "RELEASE"}}

dev/json_ld/shacl.clj

@@ -18,7 +18,6 @@
             [fluree.db.util.log :as log]
             [fluree.db.index :as index]
             [criterium.core :as criterium]
-            [clojure.data.avl :as avl]
             [clojure.tools.reader.edn :as edn]))
 
 

dev/user.clj

@@ -10,7 +10,7 @@
             [fluree.db.flake :as flake]
             [fluree.db.util.json :as json]
             [fluree.db.serde.json :as serdejson]
-            [fluree.db.storage.core :as storage]
+            [fluree.db.storage :as storage]
             [fluree.db.query.fql :as fql]
             [fluree.db.query.range :as query-range]
             [fluree.db.dbproto :as dbproto]

src/fluree/db/conn/file.cljc

@@ -10,7 +10,7 @@
             [fluree.db.conn.cache :as conn-cache]
             [fluree.db.conn.state-machine :as state-machine]
             [fluree.db.util.log :as log :include-macros true]
-            [fluree.db.storage.core :as storage]
+            [fluree.db.storage :as storage]
             [fluree.db.indexer.default :as idx-default]
             [fluree.db.serde.json :refer [json-serde]]
             #?@(:cljs [["fs" :as fs]
@@ -255,7 +255,7 @@
     [conn {:keys [id leaf tempid] :as node}]
     (let [cache-key [::resolve id tempid]]
       (if (= :empty id)
-        (storage/resolve-empty-leaf node)
+        (storage/resolve-empty-node node)
         (conn-cache/lru-lookup
           lru-cache-atom
           cache-key

src/fluree/db/conn/ipfs.cljc

@@ -1,5 +1,5 @@
 (ns fluree.db.conn.ipfs
-  (:require [fluree.db.storage.core :as storage]
+  (:require [fluree.db.storage :as storage]
             [fluree.db.index :as index]
             [fluree.db.util.context :as ctx-util]
             [fluree.db.util.core :as util :refer [exception?]]
@@ -131,7 +131,7 @@
     [conn {:keys [id leaf tempid] :as node}]
     (let [cache-key [::resolve id tempid]]
       (if (= :empty id)
-        (storage/resolve-empty-leaf node)
+        (storage/resolve-empty-node node)
         (conn-cache/lru-lookup
           lru-cache-atom
           cache-key

src/fluree/db/conn/memory.cljc

@@ -1,6 +1,6 @@
 (ns fluree.db.conn.memory
   (:require [clojure.core.async :as async :refer [go]]
-            [fluree.db.storage.core :as storage]
+            [fluree.db.storage :as storage]
             [fluree.db.index :as index]
             [fluree.db.util.context :as ctx-util]
             [fluree.db.util.core :as util]
@@ -163,7 +163,7 @@
     [_ node]
     ;; all root index nodes will be empty
 
-    (storage/resolve-empty-leaf node))
+    (storage/resolve-empty-node node))
 
   #?@(:clj
       [full-text/IndexConnection

src/fluree/db/conn/s3.clj

@@ -12,7 +12,7 @@
             [fluree.db.indexer.default :as idx-default]
             [fluree.db.ledger.proto :as ledger-proto]
             [fluree.db.serde.json :refer [json-serde]]
-            [fluree.db.storage.core :as storage]
+            [fluree.db.storage :as storage]
             [fluree.db.util.context :as ctx-util]
             [fluree.db.util.json :as json]
             [fluree.db.util.log :as log]
@@ -224,7 +224,7 @@
   (resolve [conn {:keys [id leaf tempid] :as node}]
     (let [cache-key [::resolve id tempid]]
       (if (= :empty id)
-        (storage/resolve-empty-leaf node)
+        (storage/resolve-empty-node node)
         (conn-cache/lru-lookup lru-cache-atom cache-key
                                (fn [_]
                                  (storage/resolve-index-node

src/fluree/db/db/json_ld.cljc

@@ -313,11 +313,11 @@
          opst-cmp :opst
          tspo-cmp :tspo} index/default-comparators
 
-        spot          (index/empty-branch method alias spot-cmp)
-        psot          (index/empty-branch method alias psot-cmp)
-        post          (index/empty-branch method alias post-cmp)
-        opst          (index/empty-branch method alias opst-cmp)
-        tspo          (index/empty-branch method alias tspo-cmp)
+        spot          (index/empty-branch alias spot-cmp)
+        psot          (index/empty-branch alias psot-cmp)
+        post          (index/empty-branch alias post-cmp)
+        opst          (index/empty-branch alias opst-cmp)
+        tspo          (index/empty-branch alias tspo-cmp)
         stats         {:flakes 0, :size 0, :indexed 0}
         schema        (vocab/base-schema)
         branch        (branch/branch-meta ledger)
@@ -327,7 +327,7 @@
     (map->JsonLdDb {:ledger          ledger
                     :conn            conn
                     :method          method
-                    :alias           alias
+                    :ledger-alias    alias
                     :branch          (:name branch)
                     :commit          (:commit branch)
                     :t               0

src/fluree/db/flake.cljc

@@ -1,6 +1,7 @@
 (ns fluree.db.flake
   (:refer-clojure :exclude [split-at sorted-set-by sorted-map-by take last])
-  (:require [clojure.data.avl :as avl]
+  (:require [me.tonsky.persistent-sorted-set :as pss]
+            [me.tonsky.persistent-sorted-set.arrays :as arrays]
             [fluree.db.constants :as const]
             [fluree.db.util.core :as util]
             #?(:clj [clojure.pprint :as pprint]))
@@ -227,10 +228,6 @@
   [flake]
   [(s flake) (p flake) (o flake) (dt flake) (t flake) (op flake) (m flake)])
 
-(def maximum
-  "The largest flake possible"
-  (->Flake util/max-long 0 util/max-long const/$xsd:decimal 0 true nil))
-
 (defn- assoc-flake
   "Assoc for Flakes"
   [flake k v]
@@ -451,81 +448,30 @@
 (defn slice
   "From and to are Flakes"
   [ss from to]
-  (cond
-    (and from to) (avl/subrange ss >= from <= to)
-    (nil? from) (avl/subrange ss <= to)
-    (nil? to) (avl/subrange ss >= from)
-    :else (throw (ex-info "Unexpected error performing slice, both from and to conditions are nil. Please report."
-                          {:status 500
-                           :error  :db/unexpected-error}))))
-
-(defn match-spot
-  "Returns all matching flakes to a specific subject, and optionaly also a predicate if provided
-  Must be provided with subject/predicate integer ids, no lookups are performed."
-  [ss sid pid]
-  (if pid
-    (avl/subrange ss >= (->Flake sid pid nil -1 nil nil nil)
-                  <= (->Flake sid (inc pid) nil util/max-long nil nil nil))
-    (avl/subrange ss > (->Flake (inc sid) MAX-COLL-SUBJECTS nil nil nil nil nil)
-                  < (->Flake (dec sid) -1 nil nil nil nil nil))))
-
-
-(defn match-post
-  "Returns all matching flakes to a predicate + object match."
-  [ss pid o dt]
-  (avl/subrange ss
-                >= (->Flake util/max-long pid o dt nil nil nil)
-                <= (->Flake 0 pid o dt nil nil nil)))
-
-(defn match-tspo
-  "Returns all matching flakes to a specific 't' value."
-  [ss t]
-  (avl/subrange ss
-                >= (->Flake util/max-long nil nil nil t nil nil)
-                <= (->Flake util/min-long nil nil nil t nil nil)))
-
-(defn lookup
-  [ss start-flake end-flake]
-  (avl/subrange ss >= start-flake <= end-flake))
-
-(defn subrange
-  ([ss test flake]
-   (avl/subrange ss test flake))
-  ([ss start-test start-flake end-test end-flake]
-   (avl/subrange ss start-test start-flake end-test end-flake)))
-
-
-(defn split-at
-  [n ss]
-  (avl/split-at n ss))
-
-(defn lower-than-all?
-  [f ss]
-  (let [[lower e _] (avl/split-key f ss)]
-    (and (nil? e)
-         (empty? lower))))
-
-(defn higher-than-all?
-  [f ss]
-  (let [[_ e upper] (avl/split-key f ss)]
-    (and (nil? e)
-         (empty? upper))))
-
-(defn split-by-flake
-  "Splits a sorted set at a given flake. If there is an exact match for flake,
-  puts it in the left-side. Primarily for use with last-flake."
-  [f ss]
-  (let [[l e r] (avl/split-key f ss)]
-    [(if e (conj l e) l) r]))
+  (pss/slice ss from to))
 
+(defn rslice
+  [ss to from]
+  (pss/rslice ss to from))
 
 (defn sorted-set-by
-  [comparator & flakes]
-  (apply avl/sorted-set-by comparator flakes))
+  "Create a new sorted set according to `comparator`. If a collection of flakes or
+  index nodes is supplied as the `elts` argument, the returned set will contain
+  all of the elements in the collection. If the `elts` collection is supplied,
+  it *must* already be sorted."
+  ([comparator]
+   (pss/sorted-set-by comparator))
+  ([comparator elts]
+   (->> elts
+        arrays/into-array
+        (pss/from-sorted-array comparator))))
 
-(defn sorted-map-by
-  [comparator & entries]
-  (apply avl/sorted-map-by comparator entries))
+(defn match-tspo
+  "Returns all matching flakes to a specific 't' value."
+  [ss t]
+  (pss/slice ss
+             (->Flake util/max-long nil nil nil t nil nil)
+             (->Flake util/min-long nil nil nil t nil nil)))
 
 (defn transient-reduce
   [reducer ss coll]
@@ -549,16 +495,6 @@
   [ss to-remove]
   (transient-reduce disj! ss to-remove))
 
-(defn assoc-all
-  [sm entries]
-  (transient-reduce (fn [m [k v]]
-                      (assoc! m k v))
-                    sm entries))
-
-(defn dissoc-all
-  [sm ks]
-  (transient-reduce dissoc! sm ks))
-
 (defn last
   "Returns the last item in `ss` in constant time as long as `ss` is a sorted
   set."
@@ -617,12 +553,3 @@
       (/ 1000)
       (double)
       (Math/round)))
-
-
-(defn take
-  "Takes n flakes from a sorted flake set, retaining the set itself."
-  [n flake-set]
-  (if (>= n (count flake-set))
-    flake-set
-    (let [k (nth flake-set n)]
-      (first (avl/split-key k flake-set)))))