Skip to content

Commit 7429039

Browse files
perf(aw-transform): cache categorize results per event data to fix month-view timeout (#657)
* perf(aw-transform): cache categorize results per event data to avoid redundant regex matching For a typical month with 50k+ events (from aw-watcher-window) and 20 category rules, categorize() was performing ~1M regex evaluations because every event was matched against every rule individually. Most events in a heartbeat-based watcher share identical data (same app+title). This adds an in-function HashMap cache keyed on the serialized event data JSON so that only the first occurrence of each distinct data fingerprint is matched against the rule set; subsequent identical events reuse the cached category. Expected speedup for a month-view query with 50k events and O(100) distinct app/title pairs: >99% reduction in regex work, turning a 30+ second query into a sub-second one. serde_json::Map preserves insertion order, so events produced by the same watcher in the same session produce a consistent JSON key without normalization. Adds test_categorize_cache_correctness: cache hits on identical data, correct distinct categories for differing data, and no false cache collisions. Fixes #629 * style: run cargo fmt to fix CI format check
1 parent 2de9533 commit 7429039

2 files changed

Lines changed: 151 additions & 18 deletions

File tree

Cargo.lock

Lines changed: 64 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

aw-transform/src/classify.rs

Lines changed: 87 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
use aw_models::Event;
55
use fancy_regex::Regex;
66
use lru::LruCache;
7+
use std::collections::HashMap;
78
use std::num::NonZeroUsize;
89
use std::sync::{Arc, Mutex, OnceLock};
910

@@ -117,27 +118,40 @@ impl From<Regex> for Rule {
117118
/// An event can only have one category, although the category may have a hierarchy,
118119
/// for instance: "Work -> ActivityWatch -> aw-server-rust"
119120
/// If multiple categories match, the deepest one will be chosen.
121+
///
122+
/// Performance: builds an in-memory cache keyed on the event's data JSON so that
123+
/// events with identical data (same app/title — very common in practice) are only
124+
/// matched against the rule set once. On a month's data with 50k+ events but only
125+
/// a few hundred distinct app/title pairs this reduces regex work by >99%.
120126
pub fn categorize(mut events: Vec<Event>, rules: &[(Vec<String>, Rule)]) -> Vec<Event> {
121-
let mut classified_events = Vec::new();
122-
for event in events.drain(..) {
123-
classified_events.push(categorize_one(event, rules));
127+
// Cache: serialized event data → assigned category
128+
let mut category_cache: HashMap<String, Vec<String>> = HashMap::new();
129+
let mut classified_events = Vec::with_capacity(events.len());
130+
for mut event in events.drain(..) {
131+
// Key on the full event data. serde_json::Map preserves insertion order, so
132+
// events with the same fields in the same order produce the same key — which
133+
// is the normal case for heartbeat-based watchers.
134+
let cache_key = serde_json::to_string(&event.data).unwrap_or_default();
135+
let category = category_cache
136+
.entry(cache_key)
137+
.or_insert_with(|| {
138+
let mut cat = vec!["Uncategorized".into()];
139+
for (c, rule) in rules {
140+
if rule.matches(&event) {
141+
cat = _pick_highest_ranking_category(cat, c);
142+
}
143+
}
144+
cat
145+
})
146+
.clone();
147+
event
148+
.data
149+
.insert("$category".into(), serde_json::json!(category));
150+
classified_events.push(event);
124151
}
125152
classified_events
126153
}
127154

128-
fn categorize_one(mut event: Event, rules: &[(Vec<String>, Rule)]) -> Event {
129-
let mut category: Vec<String> = vec!["Uncategorized".into()];
130-
for (cat, rule) in rules {
131-
if rule.matches(&event) {
132-
category = _pick_highest_ranking_category(category, cat);
133-
}
134-
}
135-
event
136-
.data
137-
.insert("$category".into(), serde_json::json!(category));
138-
event
139-
}
140-
141155
/// Tags a list of events
142156
///
143157
/// An event can have many tags (as opposed to only one category) which will be put into the `$tags` key of
@@ -290,6 +304,63 @@ fn test_categorize_uncategorized() {
290304
);
291305
}
292306

307+
#[test]
308+
fn test_categorize_cache_correctness() {
309+
// Verifies that the deduplication cache produces the same result as
310+
// per-event categorization when many events share the same data.
311+
let mut base = Event::default();
312+
base.data.insert("app".into(), serde_json::json!("firefox"));
313+
base.data
314+
.insert("title".into(), serde_json::json!("GitHub"));
315+
316+
let mut other = Event::default();
317+
other
318+
.data
319+
.insert("app".into(), serde_json::json!("terminal"));
320+
other.data.insert("title".into(), serde_json::json!("bash"));
321+
322+
// 50 events with same data, then 1 different event, then 50 more same
323+
let mut events: Vec<Event> = std::iter::repeat(base.clone())
324+
.take(50)
325+
.chain(std::iter::once(other.clone()))
326+
.chain(std::iter::repeat(base.clone()).take(50))
327+
.collect();
328+
329+
let rules: Vec<(Vec<String>, Rule)> = vec![
330+
(
331+
vec!["Browser".into()],
332+
Rule::Regex(RegexRule::new("firefox", true, Some(vec!["app".into()])).unwrap()),
333+
),
334+
(
335+
vec!["Terminal".into()],
336+
Rule::Regex(RegexRule::new("terminal", true, Some(vec!["app".into()])).unwrap()),
337+
),
338+
];
339+
340+
events = categorize(events, &rules);
341+
342+
assert_eq!(events.len(), 101);
343+
// All firefox events → Browser
344+
for e in events.iter().take(50) {
345+
assert_eq!(
346+
e.data.get("$category").unwrap(),
347+
&serde_json::json!(vec!["Browser"])
348+
);
349+
}
350+
// The single terminal event → Terminal
351+
assert_eq!(
352+
events[50].data.get("$category").unwrap(),
353+
&serde_json::json!(vec!["Terminal"])
354+
);
355+
// Remaining firefox events → Browser (cache hit path)
356+
for e in events.iter().skip(51) {
357+
assert_eq!(
358+
e.data.get("$category").unwrap(),
359+
&serde_json::json!(vec!["Browser"])
360+
);
361+
}
362+
}
363+
293364
#[test]
294365
fn test_tag() {
295366
let mut e = Event::default();

0 commit comments

Comments
 (0)