diff --git a/backend/migrations/0052_item_labels_tombstones.sql b/backend/migrations/0052_item_labels_tombstones.sql new file mode 100644 index 0000000..baa3071 --- /dev/null +++ b/backend/migrations/0052_item_labels_tombstones.sql @@ -0,0 +1,17 @@ +-- Tombstones for item_labels_cache +-- +-- DELETE /api/labels now soft-deletes (sets deleted_at + bumps updated_at) +-- instead of hard-deleting. This makes the `?since=` delta lossless: a removed +-- label still shows up in the delta (as a tombstone), so clients can replay +-- deletions made on other devices without a periodic full reconcile, and can +-- safely persist their delta cursor across sessions. +-- +-- NULL deleted_at = live row. A re-add (ON CONFLICT) resets deleted_at to NULL. +-- Only archived/tagged labels flow through DELETE /api/labels; `read` positions +-- are owned by the reading route and keep hard deletes. +ALTER TABLE item_labels_cache ADD COLUMN deleted_at INTEGER; + +-- Partial index to keep the hourly tombstone GC sweep cheap. +CREATE INDEX IF NOT EXISTS idx_item_labels_deleted + ON item_labels_cache(deleted_at) + WHERE deleted_at IS NOT NULL; diff --git a/backend/src/index.ts b/backend/src/index.ts index a46f753..84a1a02 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -531,6 +531,7 @@ export default { let oauthDeleted = 0; let sessionsDeleted = 0; + let labelTombstonesDeleted = 0; // Clean up expired OAuth states try { @@ -561,9 +562,25 @@ export default { console.error('[Cron] D1 WRITE ERROR deleting expired sessions:', error); } + // Purge old label tombstones (soft-deleted archived/tagged rows). The + // retention must outlast any realistic delta-cursor staleness so a + // client offline for a while still replays the deletion; 90 days mirrors + // the read-position window. deleted_at is unix seconds. + try { + const tombstoneCutoff = Math.floor(now / 1000) - 90 * 24 * 60 * 60; + const tombstoneResult = await env.DB.prepare( + 'DELETE FROM item_labels_cache WHERE deleted_at IS NOT NULL AND deleted_at < ?' + ) + .bind(tombstoneCutoff) + .run(); + labelTombstonesDeleted = tombstoneResult.meta?.changes || 0; + } catch (error) { + console.error('[Cron] D1 WRITE ERROR purging label tombstones:', error); + } + d1CleanupDuration = Date.now() - cleanupStart; console.log( - `[Cron] D1 cleanup: deleted ${oauthDeleted} OAuth states, ${sessionsDeleted} sessions, ${d1CleanupDuration}ms` + `[Cron] D1 cleanup: deleted ${oauthDeleted} OAuth states, ${sessionsDeleted} sessions, ${labelTombstonesDeleted} label tombstones, ${d1CleanupDuration}ms` ); } diff --git a/backend/src/routes/labels.ts b/backend/src/routes/labels.ts index 4e55ffc..5840b02 100644 --- a/backend/src/routes/labels.ts +++ b/backend/src/routes/labels.ts @@ -10,6 +10,7 @@ interface LabelRow { rkey: string | null; created_at: number; updated_at: number; + deleted_at: number | null; } const DEFAULT_LIMIT = 100; @@ -44,6 +45,13 @@ export async function handleGetLabels(request: Request, env: Env): Promise s.trim()) + .filter(Boolean); const itemTypeFilter = url.searchParams.get('itemType'); const cursor = url.searchParams.get('cursor'); const limitParam = url.searchParams.get('limit'); @@ -54,7 +62,7 @@ export async function handleGetLabels(request: Request, env: Env): Promise 0) { + query += ` AND label IN (${labelsFilter.map(() => '?').join(', ')})`; + params.push(...labelsFilter); + } if (itemTypeFilter) { query += ' AND item_type = ?'; params.push(itemTypeFilter); } if (Number.isFinite(since)) { + // Delta: include tombstones (deleted_at set) so the client can replay + // deletions made on other devices. The row stays until GC purges it. query += ' AND updated_at > ?'; params.push(since); + } else { + // Full snapshot: live rows only — a fresh client has nothing to remove. + query += ' AND deleted_at IS NULL'; } if (cursor) { @@ -103,6 +120,7 @@ export async function handleGetLabels(request: Request, env: Env): Promise; updatedAt: number; + deletedAt: number | null; }>; cursor?: string; }; @@ -88,6 +89,23 @@ async function getLabels(path: string): Promise<{ status: number; body: LabelsRe return { status: response.status, body }; } +// Issue a mutating request (POST add / DELETE) against /api/labels. +async function mutateLabels(method: 'POST' | 'DELETE', body: unknown): Promise { + const ctx = createExecutionContext(); + const request = new IncomingRequest('http://localhost/api/labels', { + method, + headers: { + Cookie: `session_id=${TEST_SESSION_ID}`, + Origin: env.FRONTEND_URL, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + const response = await worker.fetch(request, env, ctx); + await waitOnExecutionContext(ctx); + return response.status; +} + describe('GET /api/labels', () => { beforeEach(async () => { await env.DB.prepare('DELETE FROM item_labels_cache').run(); @@ -132,6 +150,27 @@ describe('GET /api/labels', () => { expect(body.labels.map((l) => l.itemKey)).toEqual(['b']); }); + it('restricts to a set of label types when ?labels= is provided', async () => { + const now = Math.floor(Date.now() / 1000); + await insertLabel('a', { updatedAt: now - 30, label: 'tagged' }); + await insertLabel('b', { updatedAt: now - 20, label: 'archived' }); + await insertLabel('c', { updatedAt: now - 10, label: 'read' }); + + const { body } = await getLabels('/api/labels?labels=tagged,archived'); + expect(body.labels.map((l) => l.itemKey)).toEqual(['b', 'a']); + }); + + it('excludes unrelated label types (read) from a labels-filtered delta', async () => { + const now = Math.floor(Date.now() / 1000); + await insertLabel('tag', { updatedAt: now - 10, label: 'tagged' }); + await insertLabel('readrow', { updatedAt: now - 5, label: 'read' }); + + const { body } = await getLabels( + `/api/labels?labels=tagged,archived,readProgress&since=${now - 50}` + ); + expect(body.labels.map((l) => l.itemKey)).toEqual(['tag']); + }); + describe('delta sync (?since=)', () => { it('returns only rows changed strictly after the cursor', async () => { const now = Math.floor(Date.now() / 1000); @@ -160,4 +199,54 @@ describe('GET /api/labels', () => { expect(body.labels.map((l) => l.itemKey)).toEqual(['new-tag']); }); }); + + describe('tombstones (soft delete)', () => { + it('DELETE soft-deletes: the row surfaces in a delta with deletedAt set', async () => { + const now = Math.floor(Date.now() / 1000); + await insertLabel('tag-me', { updatedAt: now - 100, label: 'tagged' }); + + expect(await mutateLabels('DELETE', { itemKey: 'tag-me', label: 'tagged' })).toBe(200); + + const { body } = await getLabels(`/api/labels?since=${now - 100}`); + expect(body.labels).toHaveLength(1); + expect(body.labels[0].itemKey).toBe('tag-me'); + expect(body.labels[0].deletedAt).toBeGreaterThan(0); + }); + + it('a full snapshot excludes tombstoned rows', async () => { + const now = Math.floor(Date.now() / 1000); + await insertLabel('keep', { updatedAt: now - 100, label: 'tagged' }); + await insertLabel('drop', { updatedAt: now - 100, label: 'tagged' }); + + expect(await mutateLabels('DELETE', { itemKey: 'drop', label: 'tagged' })).toBe(200); + + const { body } = await getLabels('/api/labels'); + expect(body.labels.map((l) => l.itemKey)).toEqual(['keep']); + }); + + it('re-adding a deleted label resurrects it (clears the tombstone)', async () => { + const now = Math.floor(Date.now() / 1000); + await insertLabel('revive', { + updatedAt: now - 100, + label: 'tagged', + props: { tags: ['a'] }, + }); + + expect(await mutateLabels('DELETE', { itemKey: 'revive', label: 'tagged' })).toBe(200); + expect( + await mutateLabels('POST', { + itemKey: 'revive', + itemType: 'article', + label: 'tagged', + props: { tags: ['b'] }, + }) + ).toBe(200); + + const { body } = await getLabels('/api/labels'); + expect(body.labels).toHaveLength(1); + expect(body.labels[0].itemKey).toBe('revive'); + expect(body.labels[0].deletedAt).toBeNull(); + expect(body.labels[0].props).toEqual({ tags: ['b'] }); + }); + }); }); diff --git a/frontend/src/lib/services/api.ts b/frontend/src/lib/services/api.ts index ed228dc..2cf6bfa 100644 --- a/frontend/src/lib/services/api.ts +++ b/frontend/src/lib/services/api.ts @@ -573,6 +573,7 @@ class ApiClient { async getLabels( options: { label?: string; + labels?: string[]; itemType?: ItemLabelType; cursor?: string; limit?: number; @@ -587,11 +588,15 @@ class ApiClient { rkey?: string; createdAt: number; updatedAt: number; + // Tombstone marker: set (unix seconds) when the label was deleted; only + // appears in delta (`since`) responses. Live snapshots never include it. + deletedAt?: number | null; }>; cursor?: string; }> { const params = new URLSearchParams(); if (options.label) params.set('label', options.label); + if (options.labels?.length) params.set('labels', options.labels.join(',')); if (options.itemType) params.set('itemType', options.itemType); if (options.cursor) params.set('cursor', options.cursor); if (options.limit) params.set('limit', String(options.limit)); @@ -601,7 +606,7 @@ class ApiClient { } async getAllLabels( - options: { label?: string; itemType?: ItemLabelType; since?: number } = {} + options: { label?: string; labels?: string[]; itemType?: ItemLabelType; since?: number } = {} ): Promise< Array<{ itemKey: string; @@ -611,6 +616,7 @@ class ApiClient { rkey?: string; createdAt: number; updatedAt: number; + deletedAt?: number | null; }> > { const all: Array<{ @@ -621,6 +627,7 @@ class ApiClient { rkey?: string; createdAt: number; updatedAt: number; + deletedAt?: number | null; }> = []; let cursor: string | undefined; do { diff --git a/frontend/src/lib/stores/itemLabels.svelte.ts b/frontend/src/lib/stores/itemLabels.svelte.ts index 6b84f4c..6741b86 100644 --- a/frontend/src/lib/stores/itemLabels.svelte.ts +++ b/frontend/src/lib/stores/itemLabels.svelte.ts @@ -1,5 +1,5 @@ import { api } from '$lib/services/api'; -import { db } from '$lib/services/db'; +import { db, getMetadata, setMetadata } from '$lib/services/db'; import { safePut, safeAdd, safeBulkPut } from '$lib/services/safeDb.svelte'; import { syncQueue, @@ -51,12 +51,17 @@ function createItemLabelsStore() { let readPositionsCursor = 0; let readPositionsFullSynced = false; - // Managed-labels (tagged/archived/readProgress) delta-sync cursor and session - // flag, mirroring the read-position pattern: the first sync does a full - // reconcile (catching deletions made on other devices), later refreshes fetch - // only rows changed since the cursor (in unix seconds). + // Managed-labels (tagged/archived/readProgress) delta-sync cursor (max + // updated_at seen, in unix seconds). Unlike read positions, this cursor is + // PERSISTED across sessions in IndexedDB: the backend tombstones deletions, so + // the delta is lossless and a cold start can resume from the saved cursor + // instead of re-fetching the whole label history. A brand-new client (no saved + // cursor) does one full snapshot to bootstrap; every sync after that is a + // delta. Hydrated once per session from `MANAGED_LABELS_CURSOR_KEY`. + const MANAGED_LABELS_CURSOR_KEY = 'managedLabelsCursor'; let managedLabelsCursor = 0; - let managedLabelsFullSynced = false; + let managedLabelsCursorLoaded = false; + let managedLabelsCursorHasValue = false; // Debounce state for batching mark-read calls let pendingMarkRead: Array<{ @@ -351,24 +356,51 @@ function createItemLabelsStore() { // Fetch managed labels in ONE paginated stream and reconcile each type, // instead of issuing a separate (paginated) /api/labels request per type. // - // The first sync of the session is a full fetch that reconciles deletions - // (tags/archives removed on other devices); subsequent refreshes fetch only - // rows changed since our cursor, keeping the common refresh tiny. As with read - // positions, cross-device DELETIONS only surface on the next full sync, since - // a delta returns updated rows and cannot observe absent ones. + // The cursor is persisted across sessions (see MANAGED_LABELS_CURSOR_KEY): a + // brand-new client does one full snapshot to bootstrap, and every sync after + // that — including cold starts — is a delta. Because the backend tombstones + // deletions, deltas carry removals too (rows with `deletedAt` set), so there + // is NO periodic full reconcile: cross-device deletes arrive in the delta. async function loadManagedLabelsFromBackend() { const managed = new Set(MANAGED_LABELS); - const isFull = !managedLabelsFullSynced; - const fetched = await api.getAllLabels(isFull ? {} : { since: managedLabelsCursor }); - // Group fetched labels by type (ignoring any non-managed labels) and track - // the newest updated_at (unix seconds) for the next delta cursor. Only the - // server-sourced timestamp is used — local labels store updatedAt in ms. + // Hydrate the persisted cursor once per session. A saved value means a prior + // session already bootstrapped the full snapshot, so we can delta from here. + if (!managedLabelsCursorLoaded) { + const persisted = await getMetadata(MANAGED_LABELS_CURSOR_KEY); + if (typeof persisted === 'number') { + managedLabelsCursor = persisted; + managedLabelsCursorHasValue = true; + } + managedLabelsCursorLoaded = true; + } + + // Restrict the fetch to our label types server-side. `item_labels_cache` + // also holds `read` rows (owned by the reading route); without this filter + // the delta would carry that read churn for the client to discard, and a big + // enough batch could spill into extra pagination round-trips. + const labels = [...MANAGED_LABELS]; + const isFull = !managedLabelsCursorHasValue; + const fetched = await api.getAllLabels( + isFull ? { labels } : { since: managedLabelsCursor, labels } + ); + + // Walk the fetched rows: track the newest updated_at (unix seconds) for the + // next cursor, apply tombstones as removals, and group live rows by type. + // Only the server-sourced timestamp is used — local labels store updatedAt + // in ms. (A full snapshot never contains tombstones — the backend filters + // them — so the deletedAt branch only fires on deltas.) let maxUpdatedAt = managedLabelsCursor; + const removed: Array<[string, string]> = []; const byLabel = new Map(); for (const raw of fetched) { if (!managed.has(raw.label)) continue; if (raw.updatedAt > maxUpdatedAt) maxUpdatedAt = raw.updatedAt; + if (raw.deletedAt != null) { + removeFromState(raw.itemKey, raw.label); + removed.push([raw.itemKey, raw.label]); + continue; + } let arr = byLabel.get(raw.label); if (!arr) { arr = []; @@ -377,24 +409,6 @@ function createItemLabelsStore() { arr.push(raw); } - // Full sync: reconcile deletions by dropping local managed labels absent - // from the complete server set. (Deltas only upsert — see note above.) - const removed: Array<[string, string]> = []; - if (isFull) { - const serverKeys = new Set(); - for (const raw of fetched) { - if (managed.has(raw.label)) serverKeys.add(makeKey(raw.itemKey, raw.label)); - } - for (const [, lbl] of labelMap) { - if (managed.has(lbl.label) && !serverKeys.has(makeKey(lbl.itemKey, lbl.label))) { - removed.push([lbl.itemKey, lbl.label]); - } - } - for (const [itemKey, label] of removed) { - removeFromState(itemKey, label); - } - } - // Upsert fetched labels (full and delta), normalising props per label type. const dbOps: ItemLabel[] = []; for (const raw of byLabel.get('tagged') || []) { @@ -455,8 +469,14 @@ function createItemLabelsStore() { console.error('Failed to sync managed labels to cache:', e); } + // Advance and persist the cursor so the next cold start resumes as a delta. managedLabelsCursor = maxUpdatedAt; - managedLabelsFullSynced = true; + managedLabelsCursorHasValue = true; + try { + await setMetadata(MANAGED_LABELS_CURSOR_KEY, managedLabelsCursor); + } catch (e) { + console.error('Failed to persist managed labels cursor:', e); + } } // --- Query methods ---