Skip to content

Commit f3d4076

Browse files
ImShyMikeImShyMike
committed
feat: db chunk time change + fix ingestion regression
1 parent b6268ce commit f3d4076

9 files changed

Lines changed: 219 additions & 61 deletions

File tree

‎frontend/src/lib/components/ui/NavLink.svelte‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -29,17 +29,20 @@
2929
3030
const resolvedHref = $derived(typeof href === 'string' ? href : resolve(...href));
3131
32-
const permissionClasses: Record<RequiredPermission, string> = {
33-
default: 'outline-surface0/75',
34-
admin:
35-
'outline-dashed bg-red/5 outline-1 outline-red focus-visible:ring-2 focus-visible:ring-red/60',
36-
owner:
37-
'outline-dashed bg-mauve/5 outline-1 outline-mauve focus-visible:ring-2 focus-visible:ring-mauve/60'
38-
};
32+
const permissionClasses = $derived.by(() => {
33+
switch (permission) {
34+
case 'admin':
35+
return `outline-dashed outline-1 outline-red focus-visible:ring-2 focus-visible:ring-red/60 ${active ? '' : 'bg-red/5'}`;
36+
case 'owner':
37+
return `outline-dashed outline-1 outline-mauve focus-visible:ring-2 focus-visible:ring-mauve/60 ${active ? '' : 'bg-mauve/5'}`;
38+
default:
39+
return active ? 'outline-1 outline-surface0' : 'outline-surface0/75';
40+
}
41+
});
3942
4043
const baseClasses =
41-
'w-full text-left outline-0 cursor-pointer py-2 rounded-md items-center inline-flex transition-colors';
42-
const activeClasses = 'bg-base outline-1 outline-surface0 text-lavender';
44+
'w-full text-left outline-0 cursor-pointer py-2 rounded-md items-center inline-flex transition-none!';
45+
const activeClasses = 'bg-base font-bold';
4346
const inactiveClasses = 'hover:bg-surface0/50 hover:outline-1';
4447
</script>
4548

@@ -48,12 +51,14 @@
4851
href={resolvedHref}
4952
{onclick}
5053
data-sveltekit-preload-data="hover"
51-
class="{baseClasses} {active ? activeClasses : inactiveClasses} {permissionClasses[
52-
permission
53-
]} {collapsed ? 'justify-center' : 'px-3'} {className}"
54+
class="{baseClasses} {active ? activeClasses : inactiveClasses} {permissionClasses} {collapsed
55+
? 'justify-center'
56+
: 'px-3'} {className}"
5457
>
5558
{#if icon}
56-
<span class="w-6 h-6 inline-flex items-center justify-center">
59+
<span
60+
class="w-6 h-6 inline-flex items-center justify-center transition-none! **:transition-none!"
61+
>
5762
{@render icon()}
5863
</span>
5964
{/if}

‎frontend/src/routes/admin/+page.svelte‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@
109109
<div class="md:col-span-2 lg:col-span-1">
110110
<StatCard
111111
title="Total Heartbeats"
112-
value={`~${adminData.total_heartbeats.toLocaleString()}`}
112+
value={adminData.total_heartbeats.toLocaleString()}
113113
valueClass="text-3xl font-bold text-ctp-green-600"
114114
/>
115115
</div>

‎rustytime/Cargo.lock‎

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rustytime/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
[package]
22
name = "rustytime-server"
33
description = "🕒 blazingly fast time tracking for developers"
4-
version = "0.22.1"
4+
version = "0.23.0"
55
edition = "2024"
66
authors = ["ImShyMike"]
77
readme = "../README.md"

‎rustytime/migrations/2026-02-22-133203_optimize_database/down.sql‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
CREATE INDEX IF NOT EXISTS idx_heartbeats_project ON heartbeats(user_id, project, time DESC) WHERE project IS NOT NULL;
2+
CREATE INDEX IF NOT EXISTS idx_heartbeats_language ON heartbeats(user_id, language, time DESC) WHERE language IS NOT NULL;
3+
CREATE INDEX IF NOT EXISTS idx_heartbeats_editor ON heartbeats(user_id, editor, time DESC) WHERE editor IS NOT NULL;
4+
CREATE INDEX IF NOT EXISTS idx_heartbeats_operating_system ON heartbeats(user_id, operating_system, time DESC) WHERE operating_system IS NOT NULL;
5+
CREATE INDEX IF NOT EXISTS idx_heartbeats_project_user ON heartbeats(time, project, user_id) WHERE project IS NOT NULL;
6+
7+
CREATE TABLE heartbeats_new (
8+
id BIGSERIAL,
9+
time TIMESTAMPTZ NOT NULL DEFAULT now(),
10+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
11+
user_id INTEGER NOT NULL,
12+
entity TEXT NOT NULL,
13+
type TEXT NOT NULL,
14+
ip_address INET NOT NULL,
15+
project TEXT,
16+
branch TEXT,
17+
language TEXT,
18+
category TEXT,
19+
is_write BOOLEAN DEFAULT FALSE,
20+
editor TEXT,
21+
operating_system TEXT,
22+
machine TEXT,
23+
user_agent TEXT NOT NULL DEFAULT '',
24+
lines INTEGER,
25+
project_root_count INTEGER,
26+
dependencies TEXT[],
27+
line_additions INTEGER,
28+
line_deletions INTEGER,
29+
lineno INTEGER,
30+
cursorpos INTEGER,
31+
source_type SMALLINT,
32+
project_id INTEGER,
33+
PRIMARY KEY (user_id, time)
34+
);
35+
36+
SELECT create_hypertable(
37+
'heartbeats_new',
38+
'time',
39+
chunk_time_interval => INTERVAL '1 day',
40+
if_not_exists => TRUE
41+
);
42+
43+
ALTER TABLE heartbeats_new SET (
44+
timescaledb.compress = true,
45+
timescaledb.compress_segmentby = 'user_id',
46+
timescaledb.compress_orderby = 'time DESC'
47+
);
48+
49+
INSERT INTO heartbeats_new (id, time, created_at, user_id, entity, type, ip_address, project, branch,
50+
language, category, is_write, editor, operating_system, machine, user_agent, lines,
51+
project_root_count, dependencies, line_additions, line_deletions, lineno, cursorpos,
52+
source_type, project_id)
53+
SELECT id, time, created_at, user_id, entity, type, ip_address, project, branch,
54+
language, category, is_write, editor, operating_system, machine, user_agent, lines,
55+
project_root_count, dependencies, line_additions, line_deletions, lineno, cursorpos,
56+
source_type, project_id
57+
FROM heartbeats;
58+
59+
SELECT setval(
60+
pg_get_serial_sequence('heartbeats_new', 'id'),
61+
(SELECT MAX(id) FROM heartbeats_new)
62+
);
63+
64+
ALTER TABLE heartbeats RENAME TO heartbeats_old;
65+
ALTER TABLE heartbeats_new RENAME TO heartbeats;
66+
67+
DROP INDEX IF EXISTS idx_heartbeats_user_time;
68+
CREATE INDEX idx_heartbeats_user_time ON heartbeats (user_id, time DESC);
69+
70+
SELECT add_compression_policy('heartbeats', INTERVAL '7 days');
71+
72+
DROP TABLE heartbeats_old CASCADE;

‎rustytime/migrations/2026-02-22-133203_optimize_database/up.sql‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
DROP INDEX IF EXISTS idx_heartbeats_project;
2+
DROP INDEX IF EXISTS idx_heartbeats_language;
3+
DROP INDEX IF EXISTS idx_heartbeats_editor;
4+
DROP INDEX IF EXISTS idx_heartbeats_operating_system;
5+
DROP INDEX IF EXISTS idx_heartbeats_project_user;
6+
7+
CREATE TABLE heartbeats_new (
8+
id BIGSERIAL,
9+
time TIMESTAMPTZ NOT NULL DEFAULT now(),
10+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
11+
user_id INTEGER NOT NULL,
12+
entity TEXT NOT NULL,
13+
type TEXT NOT NULL,
14+
ip_address INET NOT NULL,
15+
project TEXT,
16+
branch TEXT,
17+
language TEXT,
18+
category TEXT,
19+
is_write BOOLEAN DEFAULT FALSE,
20+
editor TEXT,
21+
operating_system TEXT,
22+
machine TEXT,
23+
user_agent TEXT NOT NULL DEFAULT '',
24+
lines INTEGER,
25+
project_root_count INTEGER,
26+
dependencies TEXT[],
27+
line_additions INTEGER,
28+
line_deletions INTEGER,
29+
lineno INTEGER,
30+
cursorpos INTEGER,
31+
source_type SMALLINT,
32+
project_id INTEGER,
33+
PRIMARY KEY (user_id, time)
34+
);
35+
36+
SELECT create_hypertable(
37+
'heartbeats_new',
38+
'time',
39+
chunk_time_interval => INTERVAL '1 month',
40+
if_not_exists => TRUE
41+
);
42+
43+
ALTER TABLE heartbeats_new SET (
44+
timescaledb.compress = true,
45+
timescaledb.compress_segmentby = 'user_id',
46+
timescaledb.compress_orderby = 'time DESC'
47+
);
48+
49+
INSERT INTO heartbeats_new (id, time, created_at, user_id, entity, type, ip_address, project, branch,
50+
language, category, is_write, editor, operating_system, machine, user_agent, lines,
51+
project_root_count, dependencies, line_additions, line_deletions, lineno, cursorpos,
52+
source_type, project_id)
53+
SELECT id, time, created_at, user_id, entity, type, ip_address, project, branch,
54+
language, category, is_write, editor, operating_system, machine, user_agent, lines,
55+
project_root_count, dependencies, line_additions, line_deletions, lineno, cursorpos,
56+
source_type, project_id
57+
FROM heartbeats;
58+
59+
SELECT setval(
60+
pg_get_serial_sequence('heartbeats_new', 'id'),
61+
(SELECT MAX(id) FROM heartbeats_new)
62+
);
63+
64+
ALTER TABLE heartbeats RENAME TO heartbeats_old;
65+
ALTER TABLE heartbeats_new RENAME TO heartbeats;
66+
67+
DROP INDEX IF EXISTS idx_heartbeats_user_time;
68+
CREATE INDEX idx_heartbeats_user_time ON heartbeats (user_id, time DESC);
69+
70+
SELECT add_compression_policy('heartbeats', INTERVAL '7 days');
71+
72+
SELECT 'old' AS tbl, count(*) FROM heartbeats_old
73+
UNION ALL
74+
SELECT 'new', count(*) FROM heartbeats;
75+
76+
DROP TABLE heartbeats_old CASCADE;

‎rustytime/src/handlers/api/user.rs‎

Lines changed: 46 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -264,17 +264,14 @@ async fn store_heartbeats_in_db_internal(
264264
let pool = pool.clone();
265265

266266
tokio::task::spawn_blocking(move || {
267-
let mut connection = pool.get().expect("Failed to get DB connection from pool");
267+
let mut conn = pool.get().expect("Failed to get DB connection from pool");
268268

269-
db_transaction_result!(connection, |conn| {
270-
let mut heartbeat_keys = if include_responses {
271-
Some(Vec::with_capacity(new_heartbeats.len()))
272-
} else {
273-
None
274-
};
269+
db_transaction_result!(conn, |conn| {
275270
let mut seen: HashMap<(i32, chrono::DateTime<Utc>), ()> =
276271
HashMap::with_capacity(new_heartbeats.len());
277-
let mut deduplicated = Vec::with_capacity(new_heartbeats.len());
272+
let mut ordered_keys: Vec<(i32, chrono::DateTime<Utc>)> =
273+
Vec::with_capacity(new_heartbeats.len());
274+
let mut deduplicated: Vec<NewHeartbeat> = Vec::with_capacity(new_heartbeats.len());
278275

279276
for mut heartbeat in new_heartbeats {
280277
if heartbeat.project_id.is_none()
@@ -286,47 +283,63 @@ async fn store_heartbeats_in_db_internal(
286283
}
287284

288285
let key = (heartbeat.user_id, heartbeat.time);
289-
if let Some(keys) = heartbeat_keys.as_mut() {
290-
keys.push(key);
291-
}
286+
ordered_keys.push(key);
292287

293288
if let hash_map::Entry::Vacant(e) = seen.entry(key) {
294289
e.insert(());
295290
deduplicated.push(heartbeat);
296291
}
297292
}
298293

294+
let mut inserted_map: HashMap<(i32, chrono::DateTime<Utc>), Heartbeat> = HashMap::new();
299295
let mut inserted_total = 0usize;
296+
300297
for chunk in deduplicated.chunks(HEARTBEAT_INSERT_BATCH_SIZE) {
301-
inserted_total += instrumented::execute("Heartbeat::batch_insert", || {
302-
diesel::insert_into(heartbeats::table)
303-
.values(chunk)
304-
.on_conflict((heartbeats::user_id, heartbeats::time))
305-
.do_nothing()
306-
.execute(conn)
307-
})?;
298+
let returned: Vec<Heartbeat> =
299+
instrumented::load("Heartbeat::batch_insert", || {
300+
diesel::insert_into(heartbeats::table)
301+
.values(chunk)
302+
.on_conflict((heartbeats::user_id, heartbeats::time))
303+
.do_nothing()
304+
.returning(heartbeats::all_columns)
305+
.load::<Heartbeat>(conn)
306+
})?;
307+
308+
inserted_total += returned.len();
309+
for hb in returned {
310+
inserted_map.insert((hb.user_id, hb.time), hb);
311+
}
308312
}
309313

310314
let responses = if include_responses {
311-
let unique_keys: Vec<_> = seen.keys().copied().collect();
312-
313-
let mut heartbeat_cache: HashMap<(i32, chrono::DateTime<Utc>), Heartbeat> =
314-
HashMap::new();
315-
for (uid, t) in unique_keys {
316-
let hb = instrumented::first("Heartbeat::fetch_after_insert", || {
317-
heartbeats::table
318-
.filter(heartbeats::user_id.eq(uid))
319-
.filter(heartbeats::time.eq(t))
320-
.first::<Heartbeat>(conn)
321-
})?;
322-
heartbeat_cache.insert((uid, t), hb);
315+
let missing_keys: Vec<(i32, chrono::DateTime<Utc>)> = seen
316+
.keys()
317+
.filter(|k| !inserted_map.contains_key(*k))
318+
.copied()
319+
.collect();
320+
321+
if !missing_keys.is_empty() {
322+
let user_ids: Vec<i32> = missing_keys.iter().map(|(uid, _)| *uid).collect();
323+
let times: Vec<chrono::DateTime<Utc>> =
324+
missing_keys.iter().map(|(_, t)| *t).collect();
325+
326+
let fetched: Vec<Heartbeat> =
327+
instrumented::load("Heartbeat::fetch_existing", || {
328+
heartbeats::table
329+
.filter(heartbeats::user_id.eq_any(&user_ids))
330+
.filter(heartbeats::time.eq_any(&times))
331+
.load::<Heartbeat>(conn)
332+
})?;
333+
334+
for hb in fetched {
335+
inserted_map.insert((hb.user_id, hb.time), hb);
336+
}
323337
}
324338

325-
heartbeat_keys
326-
.unwrap()
339+
ordered_keys
327340
.into_iter()
328341
.map(|key| {
329-
let hb = heartbeat_cache.get(&key).unwrap().clone();
342+
let hb = inserted_map.get(&key).unwrap().clone();
330343
HeartbeatResponse::from(hb)
331344
})
332345
.collect()

‎rustytime/src/handlers/page/admin.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ pub async fn admin_dashboard(
8888
}
8989
};
9090

91-
let total_heartbeats = db_query!(Heartbeat::total_heartbeat_count_estimate(&mut conn));
91+
let total_heartbeats = db_query!(Heartbeat::total_heartbeat_count(&mut conn));
9292
let total_users = db_query!(User::count_total_users(&mut conn, false));
9393
let heartbeats_last_hour = db_query!(Heartbeat::count_heartbeats_last_hour(&mut conn));
9494
let heartbeats_last_24h = db_query!(Heartbeat::count_heartbeats_last_24h(&mut conn));

‎rustytime/src/models/heartbeat/mod.rs‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -176,12 +176,6 @@ pub struct UserDurationRow {
176176
pub total_seconds: i64,
177177
}
178178

179-
#[derive(QueryableByName)]
180-
struct CountRow {
181-
#[diesel(sql_type = BigInt)]
182-
count: i64,
183-
}
184-
185179
#[derive(QueryableByName)]
186180
struct NullableNameDurationRow {
187181
#[diesel(sql_type = SqlNullable<Text>)]
@@ -753,12 +747,10 @@ impl From<Heartbeat> for BulkResponseItem {
753747
}
754748

755749
impl Heartbeat {
756-
pub fn total_heartbeat_count_estimate(conn: &mut PgConnection) -> QueryResult<i64> {
757-
instrumented::first("Heartbeat::count_estimate", || {
758-
diesel::sql_query("SELECT * FROM approximate_row_count('heartbeats') AS count")
759-
.get_result::<CountRow>(conn)
750+
pub fn total_heartbeat_count(conn: &mut PgConnection) -> QueryResult<i64> {
751+
instrumented::first("Heartbeat::count", || {
752+
heartbeats::table.count().get_result(conn)
760753
})
761-
.map(|res| res.count)
762754
}
763755

764756
pub fn count_heartbeats_last_24h(conn: &mut PgConnection) -> QueryResult<i64> {

0 commit comments

Comments
 (0)