Skip to content

Commit aea9f09

Browse files
tiensonqintiensonqin
authored
Simple query performance enhancements (#12262)
* perf: separate scalar from ref property query * fix: query bindings * perf: separate default value query from others * fix: import properties first and then other datoms
1 parent 045cd5f commit aea9f09

5 files changed

Lines changed: 269 additions & 154 deletions

File tree

‎deps/db/src/logseq/db/frontend/rules.cljc‎

Lines changed: 105 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -96,19 +96,11 @@
9696
[(>= ?d ?start)]
9797
[(<= ?d ?end)]]
9898

99-
:existing-property-value
100-
'[;; non-ref value
101-
[(existing-property-value ?b ?prop ?val)
102-
[?prop-e :db/ident ?prop]
103-
[(missing? $ ?prop-e :db/valueType)]
104-
[?b ?prop ?val]]
105-
;; ref value
106-
[(existing-property-value ?b ?prop ?val)
107-
[?prop-e :db/ident ?prop]
108-
[?prop-e :db/valueType :db.type/ref]
109-
[?b ?prop ?pv]
110-
(or [?pv :block/title ?val]
111-
[?pv :logseq.property/value ?val])]]
99+
:ref->val
100+
'[[(ref->val ?pv ?val)
101+
[?pv :block/title ?val]]
102+
[(ref->val ?pv ?val)
103+
[?pv :logseq.property/value ?val]]]
112104

113105
:property-missing-value
114106
'[(property-missing-value ?b ?prop-e ?default-p ?default-v)
@@ -121,31 +113,30 @@
121113
[(= ?prop-v "N/A")]
122114
[?prop-e ?default-p ?default-v]]
123115

124-
:property-scalar-default-value
125-
'[(property-scalar-default-value ?b ?prop-e ?default-p ?val)
126-
(property-missing-value ?b ?prop-e ?default-p ?default-v)
127-
[(missing? $ ?prop-e :db/valueType)]
128-
[?prop-e ?default-p ?val]]
116+
:scalar-property-value
117+
'[[(scalar-property-value ?b ?prop-e ?val)
118+
[?prop-e :db/ident ?prop]
119+
[?b ?prop ?val]]]
129120

130-
:property-default-value
131-
'[(property-default-value ?b ?prop-e ?default-p ?val)
132-
(property-missing-value ?b ?prop-e ?default-p ?default-v)
133-
(or
134-
[?default-v :block/title ?val]
135-
[?default-v :logseq.property/value ?val])]
121+
:scalar-property-value-with-default
122+
'[[(scalar-property-value-with-default ?b ?prop-e ?val)
123+
(scalar-property-value ?b ?prop-e ?val)]
124+
125+
[(scalar-property-value-with-default ?b ?prop-e ?val)
126+
(property-missing-value ?b ?prop-e :logseq.property/scalar-default-value ?val)]]
136127

137-
:property-value
138-
'[[(property-value ?b ?prop-e ?val)
128+
:ref-property-value
129+
'[[(ref-property-value ?b ?prop-e ?val)
139130
[?prop-e :db/ident ?prop]
140-
(existing-property-value ?b ?prop ?val)]
141-
[(property-value ?b ?prop-e ?val)
142-
(or
143-
(and
144-
[(missing? $ ?prop-e :db/valueType)]
145-
(property-scalar-default-value ?b ?prop-e :logseq.property/scalar-default-value ?val))
146-
(and
147-
[?prop-e :db/valueType :db.type/ref]
148-
(property-default-value ?b ?prop-e :logseq.property/default-value ?val)))]]
131+
[?b ?prop ?pv]
132+
(ref->val ?pv ?val)]]
133+
134+
:ref-property-value-with-default
135+
'[[(ref-property-value-with-default ?b ?prop-e ?val)
136+
(ref-property-value ?b ?prop-e ?val)]
137+
[(ref-property-value-with-default ?b ?prop-e ?val)
138+
(property-missing-value ?b ?prop-e :logseq.property/default-value ?pv)
139+
(ref->val ?pv ?val)]]
149140

150141
:object-has-class-property
151142
'[(object-has-class-property? ?b ?prop)
@@ -190,7 +181,65 @@
190181
[(missing? $ ?prop-e :logseq.property/public?)]
191182
[?prop-e :logseq.property/public? true])]
192183

193-
;; Checks if a property has a value for any features that are not simple queries
184+
;; Checks if a property has a value for simple queries. Supports default values
185+
:scalar-property
186+
'[(scalar-property ?b ?prop ?val)
187+
[?prop-e :db/ident ?prop]
188+
(scalar-property-value ?b ?prop-e ?val)
189+
(or
190+
[(missing? $ ?prop-e :logseq.property/public?)]
191+
[?prop-e :logseq.property/public? true])]
192+
193+
:scalar-property-with-default
194+
'[(scalar-property-with-default ?b ?prop ?val)
195+
[?prop-e :db/ident ?prop]
196+
(scalar-property-value-with-default ?b ?prop-e ?val)
197+
(or
198+
[(missing? $ ?prop-e :logseq.property/public?)]
199+
[?prop-e :logseq.property/public? true])]
200+
201+
:ref-property
202+
'[(ref-property ?b ?prop ?val)
203+
[?prop-e :db/ident ?prop]
204+
(ref-property-value ?b ?prop-e ?val)
205+
(or
206+
[(missing? $ ?prop-e :logseq.property/public?)]
207+
[?prop-e :logseq.property/public? true])]
208+
209+
:ref-property-with-default
210+
'[(ref-property-with-default ?b ?prop ?val)
211+
[?prop-e :db/ident ?prop]
212+
(ref-property-value-with-default ?b ?prop-e ?val)
213+
(or
214+
[(missing? $ ?prop-e :logseq.property/public?)]
215+
[?prop-e :logseq.property/public? true])]
216+
217+
;; Same as ref-property/scalar-property except it returns public and private properties like :block/title
218+
:private-scalar-property
219+
'[(private-scalar-property ?b ?prop ?val)
220+
[?prop-e :db/ident ?prop]
221+
[?prop-e :block/tags :logseq.class/Property]
222+
(scalar-property-value ?b ?prop-e ?val)]
223+
224+
:private-scalar-property-with-default
225+
'[(private-scalar-property-with-default ?b ?prop ?val)
226+
[?prop-e :db/ident ?prop]
227+
[?prop-e :block/tags :logseq.class/Property]
228+
(scalar-property-value-with-default ?b ?prop-e ?val)]
229+
230+
:private-ref-property
231+
'[(private-ref-property ?b ?prop ?val)
232+
[?prop-e :db/ident ?prop]
233+
[?prop-e :block/tags :logseq.class/Property]
234+
(ref-property-value ?b ?prop-e ?val)]
235+
236+
:private-ref-property-with-default
237+
'[(private-ref-property-with-default ?b ?prop ?val)
238+
[?prop-e :db/ident ?prop]
239+
[?prop-e :block/tags :logseq.class/Property]
240+
(ref-property-value-with-default ?b ?prop-e ?val)]
241+
242+
;; `property` is slow, don't use it for user-facing queries
194243
:property
195244
'[(property ?b ?prop ?val)
196245
[?prop-e :db/ident ?prop]
@@ -210,23 +259,6 @@
210259
(or [?pv :block/title ?val]
211260
[?pv :logseq.property/value ?val])))]
212261

213-
;; Checks if a property has a value for simple queries. Supports default values
214-
:simple-query-property
215-
'[(simple-query-property ?b ?prop ?val)
216-
[?prop-e :db/ident ?prop]
217-
[?prop-e :block/tags :logseq.class/Property]
218-
(or
219-
[(missing? $ ?prop-e :logseq.property/public?)]
220-
[?prop-e :logseq.property/public? true])
221-
(property-value ?b ?prop-e ?val)]
222-
223-
;; Same as property except it returns public and private properties like :block/title
224-
:private-simple-query-property
225-
'[(private-simple-query-property ?b ?prop ?val)
226-
[?prop-e :db/ident ?prop]
227-
[?prop-e :block/tags :logseq.class/Property]
228-
(property-value ?b ?prop-e ?val)]
229-
230262
:tags
231263
'[(tags ?b ?tags)
232264
[?b :block/tags ?tag]
@@ -235,34 +267,40 @@
235267

236268
:task
237269
'[(task ?b ?statuses)
238-
;; and needed to avoid binding error
239-
(and (simple-query-property ?b :logseq.property/status ?val)
240-
[(contains? ?statuses ?val)])]
270+
(ref-property-with-default ?b :logseq.property/status ?val)
271+
[(contains? ?statuses ?val)]]
241272

242273
:priority
243274
'[(priority ?b ?priorities)
244-
;; and needed to avoid binding error
245-
(and (simple-query-property ?b :logseq.property/priority ?priority)
246-
[(contains? ?priorities ?priority)])]}))
275+
(ref-property-with-default ?b :logseq.property/priority ?priority)
276+
[(contains? ?priorities ?priority)]]}))
247277

248278
(def rules-dependencies
249279
"For db graphs, a map of rule names and the rules they depend on. If this map
250280
becomes long or brittle, we could do scan rules for their deps with something
251281
like find-rules-in-where"
252282
{:has-ref #{:parent}
253283
:page-ref #{:has-ref}
254-
:task #{:simple-query-property}
255-
:priority #{:simple-query-property}
256-
:property-missing-value #{:object-has-class-property}
284+
285+
;; simple query helpers
286+
:task #{:ref-property-with-default}
287+
:priority #{:ref-property-with-default}
257288
:has-property-or-object-property #{:object-has-class-property}
258289
:object-has-class-property #{:class-extends}
259290
:has-simple-query-property #{:has-property-or-object-property}
260291
:has-private-simple-query-property #{:has-property-or-object-property}
261-
:property-default-value #{:existing-property-value :property-missing-value}
262-
:property-scalar-default-value #{:existing-property-value :property-missing-value}
263-
:property-value #{:property-default-value :property-scalar-default-value}
264-
:simple-query-property #{:property-value}
265-
:private-simple-query-property #{:property-value}})
292+
:property-missing-value #{:object-has-class-property}
293+
:ref-property-value #{:ref->val}
294+
:scalar-property #{:scalar-property-value}
295+
:scalar-property-with-default #{:scalar-property-value-with-default}
296+
:scalar-property-value-with-default #{:scalar-property-value :property-missing-value}
297+
:ref-property #{:ref-property-value}
298+
:ref-property-value-with-default #{:ref-property-value :property-missing-value}
299+
:ref-property-with-default #{:ref-property-value-with-default}
300+
:private-scalar-property #{:scalar-property-value}
301+
:private-scalar-property-with-default #{:scalar-property-value-with-default}
302+
:private-ref-property #{:ref-property-value}
303+
:private-ref-property-with-default #{:ref-property-value-with-default}})
266304

267305
(defn- get-full-deps
268306
[deps rules-deps]

‎deps/db/test/logseq/db/frontend/rules_test.cljs‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -11,17 +11,14 @@
1111
(rules/extract-rules rules/db-query-dsl-rules)))
1212

1313
(deftest get-full-deps
14-
(let [default-value-deps #{:property-default-value :property-missing-value :existing-property-value
15-
:object-has-class-property :class-extends}
16-
property-value-deps (conj default-value-deps :property-value :property-scalar-default-value)
17-
property-deps (conj property-value-deps :simple-query-property)
14+
(let [property-value-deps #{:ref->val :class-extends :object-has-class-property :property-missing-value :ref-property-value :ref-property-value-with-default}
15+
property-deps (conj property-value-deps :ref-property-with-default)
1816
task-deps (conj property-deps :task)
1917
priority-deps (conj property-deps :priority)
2018
task-priority-deps (into priority-deps task-deps)]
2119
(are [x y] (= y (#'rules/get-full-deps x rules/rules-dependencies))
22-
[:property-default-value] default-value-deps
23-
[:property-value] property-value-deps
24-
[:simple-query-property] property-deps
20+
[:ref-property-value-with-default] property-value-deps
21+
[:ref-property-with-default] property-deps
2522
[:task] task-deps
2623
[:priority] priority-deps
2724
[:task :priority] task-priority-deps)))
@@ -50,7 +47,7 @@
5047
@conn))
5148
"has-property can bind to property arg")))
5249

53-
(deftest property-rule
50+
(deftest ref-property-rule
5451
(let [conn (db-test/create-conn-with-blocks
5552
{:properties {:foo {:logseq.property/type :default}
5653
:foo2 {:logseq.property/type :default}
@@ -65,36 +62,36 @@
6562
:build/properties {:foo "bar A"}}}]})]
6663
(testing "cardinality :one property"
6764
(is (= ["Page1"]
68-
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "bar")]
65+
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "bar")]
6966
@conn)
7067
(map (comp :block/title first))))
7168
"property returns result when page has property")
7269
(is (= []
73-
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/foo "baz")]
70+
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/foo "baz")]
7471
@conn)
7572
(map (comp :block/title first))))
7673
"property returns no result when page doesn't have property value")
7774
(is (= #{:user.property/foo}
7875
(->> (q-with-rules '[:find [?p ...]
79-
:where (property ?b ?p "bar") [?b :block/title "Page1"]]
76+
:where (ref-property ?b ?p "bar") [?b :block/title "Page1"]]
8077
@conn)
8178
set))
8279
"property can bind to property arg with bound property value"))
8380

8481
(testing "cardinality :many property"
8582
(is (= ["Page1"]
86-
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 5)]
83+
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 5)]
8784
@conn)
8885
(map (comp :block/title first))))
8986
"property returns result when page has property")
9087
(is (= []
91-
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (property ?b :user.property/number-many 20)]
88+
(->> (q-with-rules '[:find (pull ?b [:block/title]) :where (ref-property ?b :user.property/number-many 20)]
9289
@conn)
9390
(map (comp :block/title first))))
9491
"property returns no result when page doesn't have property value")
9592
(is (= #{:user.property/number-many}
9693
(->> (q-with-rules '[:find [?p ...]
97-
:where (property ?b ?p 5) [?b :block/title "Page1"]]
94+
:where (ref-property ?b ?p 5) [?b :block/title "Page1"]]
9895
@conn)
9996
set))
10097
"property can bind to property arg with bound property value"))
@@ -103,7 +100,7 @@
103100
(testing ":ref property"
104101
(is (= ["Page1"]
105102
(->> (q-with-rules '[:find (pull ?b [:block/title])
106-
:where (property ?b :user.property/page-many "Page A")]
103+
:where (ref-property ?b :user.property/page-many "Page A")]
107104
@conn)
108105
(map (comp :block/title first))))
109106
"property returns result when page has property")

0 commit comments

Comments
 (0)