Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

#248 Add initial support for 'optional' queries (single statement only) #252

Merged
merged 3 commits into from
Nov 10, 2022
Merged

#248 Add initial support for 'optional' queries (single statement only) #252

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aecd21a
#248 Add initial support for 'optional' queries (single statement only)
bplatz Nov 7, 2022
b9af4ae
Remove old comment
bplatz Nov 7, 2022
0d3c08a
Update test
bplatz Nov 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 56 additions & 23 deletions src/fluree/db/query/analytical_parse.cljc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

#?(:clj (set! *warn-on-reflection* true))

(declare parse-where)
(declare parse-where parse-where-tuple)

(defn aggregate?
"Aggregate as positioned in a :select statement"
Expand Down Expand Up @@ -247,8 +247,17 @@
:filter clause-val}

:optional
{:type :optional
:where (parse-where db {:where clause-val} supplied-vars context)}
(if (vector? (first clause-val))
(if (= 1 (count clause-val))
;; single clause, just wrapped in a vector unnecessarily - still support but unwrap
(-> (apply parse-where-tuple supplied-vars supplied-vars context db (first clause-val))
(assoc :optional? true))
;; multiple optional statements, treat like a sub-query
{:type :optional
:where (parse-where db {:where clause-val} supplied-vars context)})
;; single optional statement, treat like a 3-tuple
(-> (apply parse-where-tuple supplied-vars supplied-vars context db clause-val)
(assoc :optional? true)))

:union
(if (= 2 (count clause-val))
Expand Down Expand Up @@ -1102,38 +1111,62 @@
;; once we retain the previous other ordering, add in any remaining others that might be new
(into new-others remaining-others)))}))

(defn add-where-meta-tuple
[{:keys [s p o] :as where-smt} prior-vars supplied-vars]
(let [s-var (:variable s)
p-var (:variable p)
o-var (:variable o)
s-supplied? (supplied-vars s-var)
p-supplied? (supplied-vars p-var)
o-supplied? (supplied-vars o-var)
s-out? (and s-var (not s-supplied?))
p-out? (and p-var (not p-supplied?))
o-out? (and o-var (not o-supplied?))
flake-vars [(when s-out? s-var) (when p-out? p-var) (when o-out? o-var)] ;; each var needed coming out of flake in [s p o] position
vars (get-clause-vars flake-vars prior-vars)
where-smt* (cond-> (assoc where-smt
:vars vars
:prior-vars prior-vars)
s-supplied? (assoc-in [:s :supplied?] true)
p-supplied? (assoc-in [:p :supplied?] true)
o-supplied? (assoc-in [:o :supplied?] true))]
;; return signature of all statements is vector of where statements
[where-smt* vars]))


(defn add-where-meta-optional
"Handles optional clause additional parsing."
[{:keys [where] :as optional-where-clause} prior-vars supplied-vars]
(throw (ex-info (str "Multi-statement optional clauses not yet supported!")
{:status 400 :error :db/invalid-query}))
;; TODO!
(let [where* (loop [[where-smt & r] where
prior-vars prior-vars
acc []]
(if where-smt
(let [[where-item* prior-vars*] (add-where-meta-tuple where-smt prior-vars supplied-vars)]
(recur r prior-vars* (conj acc where-item*)))
acc))
prior-vars* (-> where* last :vars)]
[(assoc optional-where-clause :where where*) prior-vars*]))


(defn add-where-meta
"Adds input vars and output vars to each where statement."
[{:keys [out-vars where select delete supplied-vars order-by group-by op-type] :as parsed-query}]
;; note: Currently 'p' is always fixed, but that is an imposed limitation - it should be able to be a variable.
(loop [[{:keys [s p o] :as where-smt} & r] where
(loop [[where-smt & r] where
i 0
prior-vars {:flake-in [] ;; variables query will need to execute
:flake-out [] ;; variables query result flakes will output
:all {} ;; cascading set of all variables used in statements through current one
:others []} ;; this is all vars minus the flake vars
acc []]
(if where-smt
(let [s-var (:variable s)
p-var (:variable p)
o-var (:variable o)
s-supplied? (supplied-vars s-var)
p-supplied? (supplied-vars p-var)
o-supplied? (supplied-vars o-var)
s-out? (and s-var (not s-supplied?))
p-out? (and p-var (not p-supplied?))
o-out? (and o-var (not o-supplied?))
flake-vars [(when s-out? s-var) (when p-out? p-var) (when o-out? o-var)] ;; each var needed coming out of flake in [s p o] position
vars (get-clause-vars flake-vars prior-vars)
where-smt* (cond-> (assoc where-smt
:i i
:vars vars
:prior-vars prior-vars)
s-supplied? (assoc-in [:s :supplied?] true)
p-supplied? (assoc-in [:p :supplied?] true)
o-supplied? (assoc-in [:o :supplied?] true))]
(recur r (inc i) vars (conj acc where-smt*)))
(let [[where-smt* prior-vars] (case (:type where-smt)
(:class :tuple :iri) (add-where-meta-tuple where-smt prior-vars supplied-vars)
:optional (add-where-meta-optional where-smt prior-vars supplied-vars))]
(recur r (inc i) prior-vars (conj acc where-smt*)))
(let [where* (where-meta-reverse acc out-vars order-by)
order-by* (update-order-by order-by group-by where*)
group-by* (update-group-by group-by where*)]
Expand Down
28 changes: 19 additions & 9 deletions src/fluree/db/query/compound.cljc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@


(defn next-chunk-s
[{:keys [conn] :as db} error-ch next-in {:keys [in-n] :as s} p idx t flake-x-form passthrough-fn]
[{:keys [conn] :as db} error-ch next-in optional? {:keys [in-n] :as s} p idx t flake-x-form passthrough-fn]
(let [out-ch (async/chan)
idx-root (get db idx)
novelty (get-in db [:novelty idx])]
Expand All @@ -56,12 +56,22 @@
(let [opts (query-range-opts idx t sid* pid nil)
in-ch (query-range/resolve-flake-slices conn idx-root novelty error-ch opts)]
;; pull all subject results off chan, push on out-ch
(loop []
(when-let [next-chunk (async/<! in-ch)]
(let [result (cond->> (sequence flake-x-form next-chunk)
pass-vals (map #(concat % pass-vals)))]
(async/>! out-ch result)
(recur))))))
(loop [interim-results nil]
(if-let [next-chunk (async/<! in-ch)]
(if (seq next-chunk)
;; calc interim results
(let [result (cond->> (sequence flake-x-form next-chunk)
pass-vals (map #(concat % pass-vals)))]
(recur (if interim-results
(into interim-results result)
result)))
;; empty result set
(or interim-results
(when optional?
(cond->> (sequence flake-x-form [(flake/parts->Flake [sid* pid])])
pass-vals (map #(concat % pass-vals))
true (async/>! out-ch)))))
(async/>! out-ch interim-results)))))
(recur r))
(async/close! out-ch))))
out-ch))
Expand All @@ -70,15 +80,15 @@
(defn get-chan
[db prev-chan error-ch clause t]
(let [out-ch (async/chan 2)
{:keys [s p o idx flake-x-form passthrough-fn]} clause
{:keys [s p o idx flake-x-form passthrough-fn optional?]} clause
{s-var :variable, s-in-n :in-n} s
{o-var :variable, o-in-n :in-n} o]
(async/go
(loop []
(if-let [next-in (async/<! prev-chan)]
(let []
(if s-in-n
(let [s-vals-chan (next-chunk-s db error-ch next-in s p idx t flake-x-form passthrough-fn)]
(let [s-vals-chan (next-chunk-s db error-ch next-in optional? s p idx t flake-x-form passthrough-fn)]
(loop []
(when-let [next-s (async/<! s-vals-chan)]
(async/>! out-ch next-s)
Expand Down
1 change: 0 additions & 1 deletion src/fluree/db/query/json_ld/response.cljc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,6 @@
acc)))))


;; TODO - check for @reverse
(defn flakes->res
"depth-i param is the depth of the graph crawl. Each successive 'ref' increases the graph depth, up to
the requested depth within the select-spec"
Expand Down
86 changes: 86 additions & 0 deletions test/fluree/db/query/optional_query_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
(ns fluree.db.query.optional-query-test
(:require
[clojure.string :as str]
[clojure.test :refer :all]
[fluree.db.test-utils :as test-utils]
[fluree.db.json-ld.api :as fluree]
[fluree.db.util.log :as log]))

(deftest ^:integration optional-queries
(testing "Testing various 'optional' query clauses."
(let [conn (test-utils/create-conn)
ledger @(fluree/create conn "query/optional" {:context {:ex "http://example.org/ns/"}})
db @(fluree/stage
ledger
[{:id :ex/brian,
:type :ex/User,
:schema/name "Brian"
:ex/friend [:ex/alice]}
{:id :ex/alice,
:type :ex/User,
:ex/favColor "Green"
:schema/email "alice@flur.ee"
:schema/name "Alice"}
{:id :ex/cam,
:type :ex/User,
:schema/name "Cam"
:schema/email "cam@flur.ee"
:ex/friend [:ex/brian :ex/alice]}])]

;; basic single optional statement
(is (= @(fluree/query db {:select ['?name '?favColor]
:where [['?s :rdf/type :ex/User]
['?s :schema/name '?name]
{:optional ['?s :ex/favColor '?favColor]}]})
[["Cam" nil]
["Alice" "Green"]
["Brian" nil]])
"Cam, Alice and Brian should all return, but only Alica has a favColor")

;; including another pass-through variable - note Brian doesn't have an email
(is (= @(fluree/query db {:select ['?name '?favColor '?email]
:where [['?s :rdf/type :ex/User]
['?s :schema/name '?name]
['?s :schema/email '?email]
{:optional ['?s :ex/favColor '?favColor]}]})
[["Cam" nil "cam@flur.ee"]
["Alice" "Green" "alice@flur.ee"]]))

;; including another pass-through variable, but with 'optional' sandwiched
(is (= @(fluree/query db {:select ['?name '?favColor '?email]
:where [['?s :rdf/type :ex/User]
['?s :schema/name '?name]
{:optional ['?s :ex/favColor '?favColor]}
['?s :schema/email '?email]]})
[["Cam" nil "cam@flur.ee"]
["Alice" "Green" "alice@flur.ee"]]))

;; query with two optionals!
(is (= @(fluree/query db {:select ['?name '?favColor '?email]
:where [['?s :rdf/type :ex/User]
['?s :schema/name '?name]
{:optional ['?s :ex/favColor '?favColor]}
{:optional ['?s :schema/email '?email]}]})
[["Cam" nil "cam@flur.ee"]
["Alice" "Green" "alice@flur.ee"]
["Brian" nil nil]]))

;; optional with unnecessary embedded vector statement
(is (= @(fluree/query db {:select ['?name '?favColor]
:where [['?s :rdf/type :ex/User]
['?s :schema/name '?name]
{:optional [['?s :ex/favColor '?favColor]]}]})
[["Cam" nil]
["Alice" "Green"]
["Brian" nil]])
"Cam, Alice and Brian should all return, but only Alica has a favColor")

;; Multiple optional clauses should work as a left outer join between them
;; TODO - not yet supported!!
(is (= (ex-message
@(fluree/query db {:select ['?name '?favColor '?email]
:where [['?s :rdf/type :ex/User]
['?s :schema/name '?name]
{:optional [['?s :ex/favColor '?favColor]
['?s :schema/email '?email]]}]}))
"Multi-statement optional clauses not yet supported!")))))
2 changes: 0 additions & 2 deletions test/fluree/db/query/reverse_query_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,4 @@
:friended {:id :ex/cam,
:rdf/type [:ex/User],
:schema/name "Cam",
:ex/last "Jones",
:schema/email "cam@example.org",
:ex/friend [{:id :ex/brian} {:id :ex/alice}]}})))))