feat(voice): TranscriptDedupe for cross-provider overlap suppression

After a mid-utterance failover the backup provider replays the ring
buffer and re-transcribes audio the primary already emitted — without
dedupe the session sees 'hello world' twice. This class filters those
duplicates using audio-clock overlap as the primary signal and
normalized string comparison as the tie-breaker (exact > superset >
fuzzy token-set similarity >= 0.85).

Same-provider observations are never considered duplicates so streaming
interim transcripts still pass through.

Task 5/17.

Author: jddunn

src/voice-pipeline/TranscriptDedupe.ts

@@ -0,0 +1,121 @@
+/**
+ * @module voice-pipeline/TranscriptDedupe
+ *
+ * Duplicate-transcript detector used by StreamingSTTChain after a mid-
+ * utterance failover. The backup provider replays the ring buffer and
+ * re-transcribes audio the primary already saw; without dedupe the session
+ * sees "hello world" twice.
+ *
+ * Dedupe signal: audio-clock overlap (primary fact) + fuzzy string match
+ * (tie-breaker). Two transcripts overlap if their [audioStartMs, audioEndMs]
+ * ranges intersect; when they do, we compare normalized text.
+ *
+ * Same-provider observations are never considered duplicates — interim
+ * transcripts from a single streaming provider are part of its protocol.
+ */
+
+export interface TranscriptObservation {
+  provider: string;
+  text: string;
+  audioStartMs: number;
+  audioEndMs: number;
+  isFinal: boolean;
+}
+
+export interface DedupeResult {
+  isDuplicate: boolean;
+  reason?: 'exact' | 'fuzzy' | 'superset';
+  against?: { provider: string; text: string };
+}
+
+const PUNCT_RE = /[.,!?;:"'()\[\]{}]/g;
+const WS_RE = /\s+/g;
+
+function normalize(text: string): string {
+  return text.toLowerCase().replace(PUNCT_RE, '').replace(WS_RE, ' ').trim();
+}
+
+function tokenSet(text: string): Set<string> {
+  return new Set(normalize(text).split(' ').filter(Boolean));
+}
+
+function tokenSetSimilarity(a: string, b: string): number {
+  const ta = tokenSet(a);
+  const tb = tokenSet(b);
+  if (ta.size === 0 && tb.size === 0) return 1;
+  if (ta.size === 0 || tb.size === 0) return 0;
+  let intersect = 0;
+  for (const t of ta) if (tb.has(t)) intersect++;
+  return intersect / Math.max(ta.size, tb.size);
+}
+
+function rangesOverlap(a: TranscriptObservation, b: TranscriptObservation): boolean {
+  return a.audioStartMs < b.audioEndMs && b.audioStartMs < a.audioEndMs;
+}
+
+export class TranscriptDedupe {
+  private recent: TranscriptObservation[] = [];
+  private readonly fuzzyThreshold: number;
+  private readonly retainMs: number;
+
+  constructor(opts: { fuzzyThreshold?: number; retainMs?: number } = {}) {
+    this.fuzzyThreshold = opts.fuzzyThreshold ?? 0.85;
+    this.retainMs = opts.retainMs ?? 10_000;
+  }
+
+  evaluate(obs: TranscriptObservation): DedupeResult {
+    this.prune(obs.audioEndMs);
+
+    for (const prev of this.recent) {
+      if (prev.provider === obs.provider) continue;
+      if (!rangesOverlap(prev, obs)) continue;
+
+      const na = normalize(prev.text);
+      const nb = normalize(obs.text);
+
+      if (na === nb) {
+        this.recent.push(obs);
+        return {
+          isDuplicate: true,
+          reason: 'exact',
+          against: { provider: prev.provider, text: prev.text },
+        };
+      }
+
+      // Longer transcript subsumes shorter one (primary saw more audio by
+      // the time backup caught up — suppress the backup's shorter view).
+      if (na.includes(nb) || nb.includes(na)) {
+        if (na.length >= nb.length) {
+          this.recent.push(obs);
+          return {
+            isDuplicate: true,
+            reason: 'superset',
+            against: { provider: prev.provider, text: prev.text },
+          };
+        }
+      }
+
+      const sim = tokenSetSimilarity(prev.text, obs.text);
+      if (sim >= this.fuzzyThreshold) {
+        this.recent.push(obs);
+        return {
+          isDuplicate: true,
+          reason: 'fuzzy',
+          against: { provider: prev.provider, text: prev.text },
+        };
+      }
+    }
+
+    this.recent.push(obs);
+    return { isDuplicate: false };
+  }
+
+  reset(): void {
+    this.recent = [];
+  }
+
+  private prune(upToMs: number): void {
+    const cutoff = upToMs - this.retainMs;
+    this.recent = this.recent.filter((o) => o.audioEndMs >= cutoff);
+  }
+}

src/voice-pipeline/__tests__/TranscriptDedupe.spec.ts

@@ -0,0 +1,135 @@
+import { describe, it, expect } from 'vitest';
+import { TranscriptDedupe } from '../TranscriptDedupe.js';
+
+describe('TranscriptDedupe', () => {
+  it('returns isDuplicate=false on first observation', () => {
+    const dedupe = new TranscriptDedupe();
+    const r = dedupe.evaluate({
+      provider: 'deepgram',
+      text: 'hello world',
+      audioStartMs: 0,
+      audioEndMs: 600,
+      isFinal: false,
+    });
+    expect(r.isDuplicate).toBe(false);
+  });
+
+  it('suppresses exact duplicates within overlap window', () => {
+    const dedupe = new TranscriptDedupe();
+    dedupe.evaluate({
+      provider: 'deepgram',
+      text: 'hello world',
+      audioStartMs: 0,
+      audioEndMs: 600,
+      isFinal: true,
+    });
+    const r = dedupe.evaluate({
+      provider: 'elevenlabs',
+      text: 'hello world',
+      audioStartMs: 100,
+      audioEndMs: 700,
+      isFinal: true,
+    });
+    expect(r.isDuplicate).toBe(true);
+    expect(r.reason).toBe('exact');
+  });
+
+  it('suppresses fuzzy matches above threshold', () => {
+    const dedupe = new TranscriptDedupe();
+    dedupe.evaluate({
+      provider: 'deepgram',
+      text: 'i said hello world',
+      audioStartMs: 0,
+      audioEndMs: 900,
+      isFinal: true,
+    });
+    const r = dedupe.evaluate({
+      provider: 'elevenlabs',
+      text: 'i said hello world.',
+      audioStartMs: 100,
+      audioEndMs: 1000,
+      isFinal: true,
+    });
+    expect(r.isDuplicate).toBe(true);
+  });
+
+  it('does NOT suppress legitimate continuations', () => {
+    const dedupe = new TranscriptDedupe();
+    dedupe.evaluate({
+      provider: 'deepgram',
+      text: 'hello world',
+      audioStartMs: 0,
+      audioEndMs: 600,
+      isFinal: true,
+    });
+    const r = dedupe.evaluate({
+      provider: 'elevenlabs',
+      text: 'how are you today',
+      audioStartMs: 800,
+      audioEndMs: 1800,
+      isFinal: true,
+    });
+    expect(r.isDuplicate).toBe(false);
+  });
+
+  it('does NOT suppress same-provider repeated transcripts', () => {
+    // Interim transcripts from the same provider shouldn't be deduped;
+    // they're a legitimate part of the streaming protocol.
+    const dedupe = new TranscriptDedupe();
+    dedupe.evaluate({
+      provider: 'deepgram',
+      text: 'hello',
+      audioStartMs: 0,
+      audioEndMs: 300,
+      isFinal: false,
+    });
+    const r = dedupe.evaluate({
+      provider: 'deepgram',
+      text: 'hello world',
+      audioStartMs: 0,
+      audioEndMs: 600,
+      isFinal: false,
+    });
+    expect(r.isDuplicate).toBe(false);
+  });
+
+  it('suppresses supersets: shorter transcript contained in longer one', () => {
+    const dedupe = new TranscriptDedupe();
+    dedupe.evaluate({
+      provider: 'deepgram',
+      text: 'the quick brown fox jumps',
+      audioStartMs: 0,
+      audioEndMs: 1500,
+      isFinal: true,
+    });
+    const r = dedupe.evaluate({
+      provider: 'elevenlabs',
+      text: 'quick brown fox',
+      audioStartMs: 200,
+      audioEndMs: 1000,
+      isFinal: true,
+    });
+    expect(r.isDuplicate).toBe(true);
+    expect(r.reason).toBe('superset');
+  });
+
+  it('reset() clears memory', () => {
+    const dedupe = new TranscriptDedupe();
+    dedupe.evaluate({
+      provider: 'a',
+      text: 'hello world',
+      audioStartMs: 0,
+      audioEndMs: 600,
+      isFinal: true,
+    });
+    dedupe.reset();
+    const r = dedupe.evaluate({
+      provider: 'b',
+      text: 'hello world',
+      audioStartMs: 0,
+      audioEndMs: 600,
+      isFinal: true,
+    });
+    expect(r.isDuplicate).toBe(false);
+  });
+});