Skip to content

Commit 60f6c76

Browse files
tiensonqintiensonqin
committed
perf: linked references with filters
1 parent 5918429 commit 60f6c76

1 file changed

Lines changed: 269 additions & 102 deletions

File tree

‎deps/db/src/logseq/db/common/reference.cljs‎

Lines changed: 269 additions & 102 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,227 @@
99
[logseq.db.common.entity-plus :as entity-plus]
1010
[logseq.db.common.initial-data :as common-initial-data]
1111
[logseq.db.frontend.class :as db-class]
12-
[logseq.db.frontend.entity-util :as entity-util]
13-
[logseq.db.frontend.rules :as rules]))
12+
[logseq.db.frontend.entity-util :as entity-util]))
13+
14+
;; -----------------------------------------------------------------------------
15+
;; Helpers (fast datoms access)
16+
;; -----------------------------------------------------------------------------
17+
18+
(defn- has-datom? [db e a v]
19+
(boolean (seq (d/datoms db :eavt e a v))))
20+
21+
(defn- entid
22+
"Normalize a datom value into an entity id.
23+
In DataScript, ref values are often raw entids (numbers), not maps."
24+
[v]
25+
(cond
26+
(nil? v) nil
27+
(number? v) v
28+
(and (map? v) (contains? v :db/id)) (:db/id v)
29+
:else v))
30+
31+
(defn- datom-v
32+
"Return the first value for attr a on entity e, normalized to entid."
33+
[db e a]
34+
(some-> (first (d/datoms db :eavt e a))
35+
:v
36+
entid))
37+
38+
(defn- datom-vs
39+
"Return all values for attr a on entity e, normalized to entids, as a set."
40+
[db e a]
41+
(into #{}
42+
(comp (map :v) (map entid))
43+
(d/datoms db :eavt e a)))
44+
45+
(defn- get-path-refs
46+
[db entity]
47+
(let [refs (mapcat :block/refs (ldb/get-block-parents db (:block/uuid entity)))
48+
block-page (:block/page entity)]
49+
(->> (cond->> refs (some? block-page) (cons block-page))
50+
distinct)))
51+
52+
(defn- get-ref-pages-count
53+
[db id ref-blocks children-ids]
54+
(when (seq ref-blocks)
55+
(let [children (->> children-ids
56+
(map (fn [id] (d/entity db id))))]
57+
(->> (concat (mapcat #(get-path-refs db %) ref-blocks)
58+
(mapcat :block/refs (concat ref-blocks children)))
59+
frequencies
60+
(keep (fn [[ref size]]
61+
(when (and (ldb/page? ref)
62+
(not= (:db/id ref) id)
63+
(not= :block/tags (:db/ident ref))
64+
(not (common-initial-data/hidden-ref? db ref id)))
65+
[(:block/title ref) size])))
66+
(sort-by second #(> %1 %2))))))
67+
68+
(defn- child-ids
69+
"Direct children via AVET on :block/parent."
70+
[db parent-eid]
71+
(into #{} (map :e) (d/datoms db :avet :block/parent parent-eid)))
72+
73+
(defn- own-refs
74+
"Refs contributed by this block itself:
75+
- direct :block/refs
76+
- implicit :block/page as a ref"
77+
[db eid]
78+
(let [direct (datom-vs db eid :block/refs)
79+
page (datom-v db eid :block/page)]
80+
(cond-> direct
81+
page (conj page))))
82+
83+
(defn- effective-refs-fn
84+
"effective-refs(eid) = own-refs(eid) ∪ effective-refs(parent(eid))"
85+
[db]
86+
(let [memo (volatile! {})]
87+
(letfn [(eff [eid]
88+
(if (contains? @memo eid)
89+
(get @memo eid)
90+
(let [own (own-refs db eid)
91+
res (if-let [p (datom-v db eid :block/parent)]
92+
(into own (eff p))
93+
own)]
94+
(vswap! memo assoc eid res)
95+
res)))]
96+
eff)))
97+
98+
(defn- allowed-subtree-refs-fn
99+
"Like subtree-refs-fn but PRUNES branches that are under an excluded ref.
100+
This implements:
101+
- parent should not be excluded just because a child has an excluded ref
102+
- BUT refs (e.g. [[bar]]) that appear under an excluded block (e.g. [[baz]])
103+
should NOT satisfy includes for ancestors.
104+
105+
Concretely: if (effective-refs node) contains any excludes, this node and its
106+
descendants contribute NOTHING to the include reachability set."
107+
[db eff excludes]
108+
(let [memo (volatile! {})]
109+
(letfn [(blocked? [eid]
110+
(and (seq excludes)
111+
(some #(contains? (eff eid) %) excludes)))
112+
(sub [eid]
113+
(if (contains? @memo eid)
114+
(get @memo eid)
115+
(let [res (if (blocked? eid)
116+
#{}
117+
(reduce (fn [acc c] (into acc (sub c)))
118+
(own-refs db eid)
119+
(child-ids db eid)))]
120+
(vswap! memo assoc eid res)
121+
res)))]
122+
sub)))
123+
124+
(defn- matches-filters?
125+
"Include semantics: AND (must all be present) against include-set.
126+
Exclude semantics: NONE (must not contain any) against exclude-set."
127+
[include-set exclude-set includes excludes]
128+
(and
129+
(or (empty? includes)
130+
(every? #(contains? include-set %) includes))
131+
(or (empty? excludes)
132+
(not (some #(contains? exclude-set %) excludes)))))
133+
134+
(defn- matched-ref-block-ids-under-top
135+
"Return the set of block ids (top or child) that match filters.
136+
137+
Semantics (matches your example):
138+
- Includes (AND) may be satisfied by descendants, BUT only from descendants
139+
that are NOT under an excluded ref.
140+
Example:
141+
- [[foo]]
142+
- [[baz]]
143+
- [[bar]]
144+
With include=bar, exclude=baz => foo should NOT match because bar is under baz.
145+
146+
- Excludes are checked ONLY against effective-refs(node) (self + parents),
147+
so a parent isn't excluded just because a child mentions an excluded page.
148+
149+
Pruning:
150+
- If includes cannot be satisfied anywhere below (effective + allowed-subtree),
151+
prune subtree."
152+
[db top-ref-block-ids includes excludes class-ids]
153+
(let [eff (effective-refs-fn db)
154+
;; allowed subtree refs for includes (prunes excluded branches)
155+
allowed-subrefs (allowed-subtree-refs-fn db eff (->> excludes (remove nil?) vec))
156+
includes (->> includes (remove nil?) vec)
157+
excludes (->> excludes (remove nil?) vec)
158+
class-ids (when (seq class-ids) (->> class-ids (remove nil?) vec))
159+
160+
can-satisfy-includes? (fn [eff-refs node]
161+
(or (empty? includes)
162+
(let [possible (into eff-refs (allowed-subrefs node))]
163+
(every? #(contains? possible %) includes))))
164+
165+
class-ok? (fn [eid]
166+
(or (empty? class-ids)
167+
(not (some #(has-datom? db eid :block/tags %) class-ids))))]
168+
169+
(loop [stack (vec top-ref-block-ids)
170+
visited #{}
171+
out #{}]
172+
(if (empty? stack)
173+
out
174+
(let [eid (peek stack)
175+
stack (pop stack)]
176+
(if (contains? visited eid)
177+
(recur stack visited out)
178+
(let [visited (conj visited eid)
179+
eff-refs (eff eid)]
180+
(cond
181+
;; don't match this node, but still traverse
182+
(not (class-ok? eid))
183+
(recur (into stack (child-ids db eid)) visited out)
184+
185+
;; prune: AND includes cannot be satisfied anywhere below
186+
(not (can-satisfy-includes? eff-refs eid))
187+
(recur stack visited out)
188+
189+
:else
190+
(let [include-set (into eff-refs (allowed-subrefs eid))
191+
exclude-set eff-refs
192+
out' (if (matches-filters? include-set exclude-set includes excludes)
193+
(conj out eid)
194+
out)]
195+
(recur (into stack (child-ids db eid)) visited out'))))))))))
196+
197+
;; -----------------------------------------------------------------------------
198+
;; Expand matched refs up to top refs
199+
;; -----------------------------------------------------------------------------
200+
201+
(defn- expand-to-top-refs
202+
"Given matched refs, add ancestors until reaching a top ref in `top-ref-ids`.
203+
Uses cached parent lookups (datoms only)."
204+
[db top-ref-ids matched-ref-ids]
205+
(let [parent-cache (volatile! {}) ;; eid -> parent-eid|nil
206+
result (volatile! #{})
207+
top? (fn [eid] (contains? top-ref-ids eid))
208+
parent-of (fn [eid]
209+
(if (contains? @parent-cache eid)
210+
(get @parent-cache eid)
211+
(let [p (some-> (first (d/datoms db :eavt eid :block/parent))
212+
:v
213+
entid)]
214+
(vswap! parent-cache assoc eid p)
215+
p)))]
216+
(doseq [start matched-ref-ids]
217+
(loop [eid start]
218+
(when eid
219+
(cond
220+
(contains? @result eid)
221+
nil
222+
(top? eid)
223+
(vswap! result conj eid)
224+
:else
225+
(do
226+
(vswap! result conj eid)
227+
(recur (parent-of eid)))))))
228+
@result))
229+
230+
;; -----------------------------------------------------------------------------
231+
;; Public API
232+
;; -----------------------------------------------------------------------------
14233

15234
(defn get-filters
16235
[db page]
@@ -37,108 +256,55 @@
37256
(catch :default e
38257
(log/error :syntax/filters e)))))))
39258

40-
(defn- build-include-exclude-query
41-
[includes excludes]
42-
(concat
43-
(for [include includes]
44-
(list 'has-ref '?b include))
45-
(for [exclude excludes]
46-
(list 'not (list 'has-ref '?b exclude)))))
47-
48-
(defn- filter-refs-query
49-
[includes excludes class-ids]
50-
(let [clauses (concat
51-
(build-include-exclude-query includes excludes)
52-
(for [class-id class-ids]
53-
(list 'not ['?b :block/tags class-id])))]
54-
(into [:find '[?b ...]
55-
:in '$ '% '[?id ...]
56-
:where
57-
(list 'has-ref '?b '?id)]
58-
clauses)))
259+
(defn get-linked-references [db id]
260+
(let [entity (d/entity db id)
261+
ids (set (cons id (ldb/get-block-alias db id)))
262+
page-filters (get-filters db entity)
263+
excludes (map :db/id (:excluded page-filters))
264+
includes (map :db/id (:included page-filters))
265+
has-filters? (or (seq excludes) (seq includes))
59266

60-
(defn- get-path-refs
61-
[db entity]
62-
(let [refs (mapcat :block/refs (ldb/get-block-parents db (:block/uuid entity)))
63-
block-page (:block/page entity)]
64-
(->> (cond->> refs (some? block-page) (cons block-page))
65-
distinct)))
267+
class-ids (when (ldb/class? entity)
268+
(let [class-children (db-class/get-structured-children db id)]
269+
(set (conj class-children id))))
66270

67-
(defn- get-ref-pages-count
68-
[db id ref-blocks children-ids]
69-
(when (seq ref-blocks)
70-
(let [children (->> children-ids
71-
(map (fn [id] (d/entity db id))))]
72-
(->> (concat (mapcat #(get-path-refs db %) ref-blocks)
73-
(mapcat :block/refs (concat ref-blocks children)))
74-
frequencies
75-
(keep (fn [[ref size]]
76-
(when (and (ldb/page? ref)
77-
(not= (:db/id ref) id)
78-
(not= :block/tags (:db/ident ref))
79-
(not (common-initial-data/hidden-ref? db ref id)))
80-
[(:block/title ref) size])))
81-
(sort-by second #(> %1 %2))))))
271+
;; Collect all top ref blocks that directly reference the page (or any alias).
272+
full-ref-block-ids
273+
(->> ids
274+
(mapcat (fn [pid] (:block/_refs (d/entity db pid))))
275+
(remove (fn [ref]
276+
(or
277+
(when class-ids
278+
(some class-ids (map :db/id (:block/tags ref))))
279+
(entity-util/hidden? ref)
280+
(entity-util/hidden? (:block/page ref)))))
281+
(map :db/id)
282+
set)
82283

83-
(defn- get-block-parents-until-top-ref
84-
[db id ref-id ref-block-ids *result]
85-
(loop [eid ref-id
86-
parents' []]
87-
(when eid
88-
(cond
89-
(contains? @*result eid)
90-
(swap! *result into parents')
91-
92-
(contains? ref-block-ids eid)
93-
(when-not (common-initial-data/hidden-ref? db (d/entity db eid) id)
94-
(swap! *result into (conj parents' eid)))
95-
:else
96-
(let [e (d/entity db eid)]
97-
(recur (:db/id (:block/parent e)) (conj parents' eid)))))))
98-
99-
;; TODO(perf): recursive datascript rule is still too slow for filters for large graphs
100-
(defn get-linked-references
101-
[db id]
102-
(let [entity (d/entity db id)
103-
ids (set (cons id (ldb/get-block-alias db id)))
104-
page-filters (get-filters db entity)
105-
excludes (map :db/id (:excluded page-filters))
106-
includes (map :db/id (:included page-filters))
107-
has-filters? (or (seq excludes) (seq includes))
108-
class-ids (when (ldb/class? entity)
109-
(let [class-children (db-class/get-structured-children db id)]
110-
(set (conj class-children id))))
111-
full-ref-block-ids (->> ids
112-
(mapcat (fn [id] (:block/_refs (d/entity db id))))
113-
(remove (fn [ref]
114-
(or
115-
(when class-ids
116-
(some class-ids (map :db/id (:block/tags ref))))
117-
(entity-util/hidden? ref)
118-
(entity-util/hidden? (:block/page ref)))))
119-
(map :db/id)
120-
set)
121-
matched-ref-block-ids (when has-filters?
122-
(let [ref-ids (d/q (filter-refs-query includes excludes class-ids)
123-
db
124-
(rules/extract-rules rules/db-query-dsl-rules
125-
[:has-ref]
126-
{:deps rules/rules-dependencies})
127-
ids)]
128-
(set ref-ids)))
129-
matched-refs-with-children-ids (when has-filters?
130-
(let [*result (atom #{})]
131-
(doseq [ref-id matched-ref-block-ids]
132-
(get-block-parents-until-top-ref db id ref-id full-ref-block-ids *result))
133-
@*result))
134-
ref-blocks (->> (if has-filters?
135-
(set/intersection full-ref-block-ids matched-refs-with-children-ids)
136-
full-ref-block-ids)
137-
(map (fn [id] (d/entity db id))))
138-
children-ids (if has-filters?
139-
(set (remove full-ref-block-ids matched-refs-with-children-ids))
140-
(->> (mapcat (fn [ref] (ldb/get-block-children-ids db (:db/id ref))) ref-blocks)
141-
set))]
284+
;; matched can be top or child ids
285+
matched-ref-block-ids
286+
(when has-filters?
287+
(matched-ref-block-ids-under-top db full-ref-block-ids includes excludes class-ids))
288+
289+
;; Expand matches up to top refs so we can show parent chains and matched children.
290+
matched-refs-with-children-ids
291+
(when has-filters?
292+
(expand-to-top-refs db full-ref-block-ids matched-ref-block-ids))
293+
294+
final-ref-ids
295+
(if has-filters?
296+
(set/intersection full-ref-block-ids matched-refs-with-children-ids)
297+
full-ref-block-ids)
298+
299+
;; Materialize only at the end.
300+
ref-blocks (map #(d/entity db %) final-ref-ids)
301+
302+
children-ids
303+
(if has-filters?
304+
(set (remove full-ref-block-ids matched-refs-with-children-ids))
305+
(->> ref-blocks
306+
(mapcat (fn [ref] (ldb/get-block-children-ids db (:db/id ref))))
307+
set))]
142308
{:ref-blocks ref-blocks
143309
:ref-pages-count (get-ref-pages-count db id ref-blocks children-ids)
144310
:ref-matched-children-ids (when has-filters? children-ids)}))
@@ -150,7 +316,8 @@
150316
(when-not (string/blank? title)
151317
(let [ids (->> (d/datoms db :avet :block/title)
152318
(keep (fn [d]
153-
(when (and (not= id (:e d)) (string/includes? (string/lower-case (:v d)) title))
319+
(when (and (not= id (:e d))
320+
(string/includes? (string/lower-case (:v d)) title))
154321
(:e d)))))]
155322
(keep
156323
(fn [eid]

0 commit comments

Comments
 (0)