Speed up spam classifier neighbor lookup on hot bands

Author: prasmussen

glot_backend/priv/db/migrations/0011_spam_classifier_band_eligibility.sql

@@ -0,0 +1,9 @@
+-- Band rows only exist for neighbor-eligible fingerprints (token_count >= 20).
+-- Shorter snippets never pass the similarity threshold, and near-empty ones
+-- share a degenerate signature that made every lookup scan the same hot bands.
+DELETE FROM spam_classifier_bands b
+USING spam_classifier_fingerprints f
+WHERE f.snippet_id = b.snippet_id
+  AND f.content_revision = b.content_revision
+  AND f.algorithm_version = b.algorithm_version
+  AND f.token_count < 20;

glot_backend/src/glot_backend/spam_classifier/domain/similarity.gleam

@@ -81,7 +81,24 @@ pub fn jaccard(left: Fingerprint, right: Fingerprint) -> Float {
 }
 
 pub fn is_neighbor(left: Fingerprint, right: Fingerprint) -> Bool {
-  left.token_count >= 20
-  && right.token_count >= 20
+  can_have_neighbors(left)
+  && can_have_neighbors(right)
   && jaccard(left, right) >=. 0.8
 }
+
+const neighbor_token_threshold = 20
+
+/// Only fingerprints at or above the token threshold take part in neighbor
+/// matching. Shorter snippets have too little content to compare, and the
+/// near-empty ones share a degenerate signature that would form hot bands.
+pub fn can_have_neighbors(fingerprint: Fingerprint) -> Bool {
+  fingerprint.token_count >= neighbor_token_threshold
+}
+
+/// Bands to look up when searching for neighbors: none for short snippets.
+pub fn candidate_bands(fingerprint: Fingerprint) -> List(String) {
+  case can_have_neighbors(fingerprint) {
+    True -> fingerprint.bands
+    False -> []
+  }
+}

glot_backend/src/glot_backend/spam_classifier/effect/classification.gleam

@@ -71,7 +71,7 @@ pub fn run(
     classifier_provider.Local -> {
       use candidates <- result.try(ports.storage.candidates(
         scoring.version,
-        signature.bands,
+        similarity.candidate_bands(signature),
         request.snippet.id,
       ))
       let verified =

glot_backend/src/glot_backend/spam_classifier/sql/fingerprints.sql

@@ -14,32 +14,43 @@ WITH current_snippet AS (
     trigram_hashes = EXCLUDED.trigram_hashes, signature = EXCLUDED.signature,
     independently_suspicious = EXCLUDED.independently_suspicious,
     urls = EXCLUDED.urls, indexed_at = CURRENT_TIMESTAMP
-  RETURNING snippet_id, content_revision, algorithm_version
+  RETURNING snippet_id, content_revision, algorithm_version, token_count
 ), bands AS (
+  -- Only neighbor-eligible snippets get band rows: shorter ones never pass
+  -- the token threshold, and their degenerate signatures would form hot bands.
   INSERT INTO spam_classifier_bands
     (snippet_id, content_revision, algorithm_version, band_number, band_value)
   SELECT stored.snippet_id, stored.content_revision, stored.algorithm_version,
     (band.ordinality - 1)::int, band.value
   FROM stored CROSS JOIN unnest(sqlc.arg(bands)::text[]) WITH ORDINALITY AS band(value, ordinality)
+  WHERE stored.token_count >= 20
   ON CONFLICT (snippet_id, content_revision, algorithm_version, band_number)
   DO UPDATE SET band_value = EXCLUDED.band_value
 )
 SELECT EXISTS(SELECT 1 FROM stored)::boolean AS stored;
 
 -- name: FindClassifierNeighbors :many
+-- Rank on the narrow band rows first; identical content shares every band, so
+-- hot band values can match tens of thousands of snippets. Joining the wide
+-- fingerprint rows before the LIMIT forced a large on-disk sort.
+WITH ranked AS (
+  SELECT b.snippet_id, b.content_revision, b.algorithm_version,
+    count(*)::int AS matching_bands
+  FROM spam_classifier_bands b
+  WHERE b.algorithm_version = sqlc.arg(algorithm_version)::text
+    AND b.band_value = ANY(sqlc.arg(bands)::text[])
+    AND b.snippet_id <> sqlc.arg(snippet_id)::uuid
+  GROUP BY b.snippet_id, b.content_revision, b.algorithm_version
+  ORDER BY matching_bands DESC, b.content_revision DESC, b.snippet_id DESC
+  LIMIT 200
+)
 SELECT f.snippet_id, f.content_revision, s.slug, f.token_count,
-  f.trigram_hashes, f.signature, f.independently_suspicious,
-  count(*)::int AS matching_bands
-FROM spam_classifier_bands b
+  f.trigram_hashes, f.signature, f.independently_suspicious, r.matching_bands
+FROM ranked r
 JOIN spam_classifier_fingerprints f USING (snippet_id, content_revision, algorithm_version)
 JOIN snippets s ON s.id = f.snippet_id AND s.updated_at = f.content_revision
-WHERE b.algorithm_version = sqlc.arg(algorithm_version)::text
-  AND b.band_value = ANY(sqlc.arg(bands)::text[])
-  AND b.snippet_id <> sqlc.arg(snippet_id)::uuid
-  AND f.token_count >= 20
-GROUP BY f.snippet_id, f.content_revision, f.algorithm_version, s.slug
-ORDER BY matching_bands DESC, f.content_revision DESC, f.snippet_id DESC
-LIMIT 200;
+WHERE f.token_count >= 20
+ORDER BY r.matching_bands DESC, f.content_revision DESC, f.snippet_id DESC;
 
 -- name: ListClassifierIndexBatch :many
 SELECT s.id, s.slug, s.user_id, s.language, s.title, s.visibility, s.stdin,
@@ -106,6 +117,7 @@ WITH progress AS MATERIALIZED (
     (band.ordinality - 1)::int, band.value
   FROM stored JOIN input i ON i.snippet_id = stored.snippet_id
   CROSS JOIN unnest(i.bands) WITH ORDINALITY AS band(value, ordinality)
+  WHERE i.token_count >= 20
   ON CONFLICT (snippet_id, content_revision, algorithm_version, band_number)
   DO UPDATE SET band_value = EXCLUDED.band_value
 ), advanced AS (

glot_backend/src/glot_backend/sql.gleam

@@ -6918,13 +6918,16 @@ pub fn store_classifier_fingerprint(
     trigram_hashes = EXCLUDED.trigram_hashes, signature = EXCLUDED.signature,
     independently_suspicious = EXCLUDED.independently_suspicious,
     urls = EXCLUDED.urls, indexed_at = CURRENT_TIMESTAMP
-  RETURNING snippet_id, content_revision, algorithm_version
+  RETURNING snippet_id, content_revision, algorithm_version, token_count
 ), bands AS (
+  -- Only neighbor-eligible snippets get band rows: shorter ones never pass
+  -- the token threshold, and their degenerate signatures would form hot bands.
   INSERT INTO spam_classifier_bands
     (snippet_id, content_revision, algorithm_version, band_number, band_value)
   SELECT stored.snippet_id, stored.content_revision, stored.algorithm_version,
     (band.ordinality - 1)::int, band.value
   FROM stored CROSS JOIN unnest($9::text[]) WITH ORDINALITY AS band(value, ordinality)
+  WHERE stored.token_count >= 20
   ON CONFLICT (snippet_id, content_revision, algorithm_version, band_number)
   DO UPDATE SET band_value = EXCLUDED.band_value
 )
@@ -6972,19 +6975,24 @@ pub fn find_classifier_neighbors(
   snippet_id snippet_id: BitArray,
 ) {
   let sql =
-    "SELECT f.snippet_id, f.content_revision, s.slug, f.token_count,
-  f.trigram_hashes, f.signature, f.independently_suspicious,
-  count(*)::int AS matching_bands
-FROM spam_classifier_bands b
+    "WITH ranked AS (
+  SELECT b.snippet_id, b.content_revision, b.algorithm_version,
+    count(*)::int AS matching_bands
+  FROM spam_classifier_bands b
+  WHERE b.algorithm_version = $1::text
+    AND b.band_value = ANY($2::text[])
+    AND b.snippet_id <> $3::uuid
+  GROUP BY b.snippet_id, b.content_revision, b.algorithm_version
+  ORDER BY matching_bands DESC, b.content_revision DESC, b.snippet_id DESC
+  LIMIT 200
+)
+SELECT f.snippet_id, f.content_revision, s.slug, f.token_count,
+  f.trigram_hashes, f.signature, f.independently_suspicious, r.matching_bands
+FROM ranked r
 JOIN spam_classifier_fingerprints f USING (snippet_id, content_revision, algorithm_version)
 JOIN snippets s ON s.id = f.snippet_id AND s.updated_at = f.content_revision
-WHERE b.algorithm_version = $1::text
-  AND b.band_value = ANY($2::text[])
-  AND b.snippet_id <> $3::uuid
-  AND f.token_count >= 20
-GROUP BY f.snippet_id, f.content_revision, f.algorithm_version, s.slug
-ORDER BY matching_bands DESC, f.content_revision DESC, f.snippet_id DESC
-LIMIT 200"
+WHERE f.token_count >= 20
+ORDER BY r.matching_bands DESC, f.content_revision DESC, f.snippet_id DESC"
   #(
     sql,
     [
@@ -7175,6 +7183,7 @@ pub fn commit_classifier_index_batch(
     (band.ordinality - 1)::int, band.value
   FROM stored JOIN input i ON i.snippet_id = stored.snippet_id
   CROSS JOIN unnest(i.bands) WITH ORDINALITY AS band(value, ordinality)
+  WHERE i.token_count >= 20
   ON CONFLICT (snippet_id, content_revision, algorithm_version, band_number)
   DO UPDATE SET band_value = EXCLUDED.band_value
 ), advanced AS (

glot_backend/test/spam_classifier/domain/features_test.gleam

@@ -67,6 +67,12 @@ pub fn url_substitutions_have_identical_fingerprints_test() {
 pub fn short_snippets_never_qualify_test() {
   let short = similarity.fingerprint(["hello", "world", "example"])
   assert similarity.is_neighbor(short, short) == False
+  assert similarity.can_have_neighbors(short) == False
+  assert similarity.candidate_bands(short) == []
+  let long =
+    similarity.fingerprint(list.repeat("word", 19) |> list.append(["end"]))
+  assert similarity.can_have_neighbors(long)
+  assert similarity.candidate_bands(long) == long.bands
 }
 
 pub fn defanged_protocol_and_hidden_bare_domain_are_obfuscation_test() {