From 9f16959339e0bf854c811203848fc02c7b0a1bfd Mon Sep 17 00:00:00 2001 From: James Kane Date: Sun, 2 Aug 2026 09:48:43 -0500 Subject: [PATCH 01/10] Return the caller's own sample guid with an IBD suggestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `/api/v1/ibd/attest` gates on `owns_sample(attester_did, claimed_sample)`, but a self-publishing Edge client has no way to learn its server-side `core.biosample.sample_guid` — the suggestions payload returned only the *candidate's* guid, so Navigator could never fill in `claimed_sample` and the attest endpoint was unreachable from the edge. `suggestions_for_did` already joins on `ms.target_sample_guid` (that is how the per-DID scope is enforced), so surfacing it costs nothing and leaks nothing: the caller owns that sample by construction. `suggested_sample_guid` supplies the `counterpart_sample` of the same report. Co-Authored-By: Claude Opus 5 (1M context) --- rust/crates/du-db/src/ibd.rs | 10 +++++++--- rust/crates/du-db/tests/ibd_suggestions.rs | 3 +++ rust/crates/du-web/src/routes/ibd.rs | 3 +++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/rust/crates/du-db/src/ibd.rs b/rust/crates/du-db/src/ibd.rs index 02733106..5caaaa52 100644 --- a/rust/crates/du-db/src/ibd.rs +++ b/rust/crates/du-db/src/ibd.rs @@ -77,9 +77,13 @@ pub struct SuggestionReport { pub suggestions_written: u64, } -/// A ranked suggestion for a sample (the reader's row). +/// A ranked suggestion for a sample (the reader's row). `target_sample_guid` is the reader's +/// **own** sample the candidate was matched against — the caller already owns it, so returning +/// it reveals nothing new, and the Edge needs it as the `claimed_sample` of an +/// [`messages::attest`] report (which [`record_attestation`] gates on ownership). #[derive(Debug, Clone, sqlx::FromRow)] pub struct SuggestionView { + pub target_sample_guid: Uuid, pub suggested_sample_guid: Uuid, pub suggestion_type: String, pub score: Option, @@ -89,7 +93,7 @@ pub struct SuggestionView { /// Serve a sample's ranked active candidates (used by the eventual consent-gated API). pub async fn suggestions_for(pool: &PgPool, sample_guid: Uuid, limit: i64) -> Result, DbError> { Ok(sqlx::query_as( - "SELECT suggested_sample_guid, suggestion_type, score, metadata \ + "SELECT target_sample_guid, suggested_sample_guid, suggestion_type, score, metadata \ FROM ibd.match_suggestion \ WHERE target_sample_guid = $1 AND status = 'ACTIVE' \ ORDER BY score DESC NULLS LAST LIMIT $2", @@ -133,7 +137,7 @@ pub mod messages { /// a counterpart DID (identity reveal stays Edge-to-Edge over D1 consent). pub async fn suggestions_for_did(pool: &PgPool, did: &str, limit: i64) -> Result, DbError> { Ok(sqlx::query_as( - "SELECT ms.suggested_sample_guid, ms.suggestion_type, ms.score, ms.metadata \ + "SELECT ms.target_sample_guid, ms.suggested_sample_guid, ms.suggestion_type, ms.score, ms.metadata \ FROM ibd.match_suggestion ms \ JOIN core.biosample b ON b.sample_guid = ms.target_sample_guid \ WHERE b.atproto->>'repo_did' = $1 AND ms.status = 'ACTIVE' \ diff --git a/rust/crates/du-db/tests/ibd_suggestions.rs b/rust/crates/du-db/tests/ibd_suggestions.rs index 240b35e7..c2fbe714 100644 --- a/rust/crates/du-db/tests/ibd_suggestions.rs +++ b/rust/crates/du-db/tests/ibd_suggestions.rs @@ -198,6 +198,9 @@ async fn suggestions_scoped_by_owner_did() { let mine = ibd::suggestions_for_did(&pool, "did:ex:owner", 50).await.unwrap(); assert_eq!(mine.len(), 1); assert_eq!(mine[0].suggested_sample_guid, suggested); + // The row also names the caller's OWN sample — the Edge attests with it as `claimed_sample`, + // and it is the only way a self-publishing client learns its server-side sample guid. + assert_eq!(mine[0].target_sample_guid, target); assert!(ibd::suggestions_for_did(&pool, "did:ex:counterpart", 50).await.unwrap().is_empty()); // Introduce authorization: true only for the owner's genuine candidate. diff --git a/rust/crates/du-web/src/routes/ibd.rs b/rust/crates/du-web/src/routes/ibd.rs index 332ea72d..ae303202 100644 --- a/rust/crates/du-web/src/routes/ibd.rs +++ b/rust/crates/du-web/src/routes/ibd.rs @@ -45,6 +45,9 @@ async fn suggestions(State(st): State, Query(q): Query Date: Tue, 4 Aug 2026 07:07:47 -0500 Subject: [PATCH 02/10] fix(tree): one-off script to fill Y tree branch-row build coordinates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The de-novo loader reuses a catalog row only when it matches on `coordinates @> {'hs1': ...}`, so markers whose hs1 coordinate had not been lifted yet missed the match and got a fresh hs1-only row — and that row is what `tree.haplogroup_variant` points at. `variant-name-reconcile` later adopted the marker's name onto the branch row but never touches `coordinates`, leaving the tree with correct names and, on the 2026-08-04 prod dump, GRCh38 for only 44,181 of 203,983 branch SNPs (21.7%). The Navigator places each source in its native build with no liftover (`place_y_consensus_decodingus`), and `parse_decodingus_json(json, build_key)` drops every locus lacking that build's coordinate — so a GRCh38 subject saw 3,413 of 11,421 Y nodes. All 33 backbone nodes survived; what vanished was the terminal tree (node visibility 81% at depth 0-10 → 21% at 31-40), so GRCh38 subjects placed plausibly but shallow rather than failing outright. Copy GRCh38/GRCh37 from the marker's catalog row (same canonical_name AND identical hs1 site + alleles) rather than chain-lifting: the catalog row holds YBrowse's own values, and a lift would re-derive them and can mismap in the inverted / ampliconic Y blocks. All 132,183 candidate rows agree with their twin on position and alleles (0 swapped), so the copy is exact. Rows whose twin disagrees are not filled — the match is enforced in the join. Verified against a restored copy of the 2026-08-04 prod dump and against decodingus_cutover: 130,553 rows filled in 34s, GRCh38 44,181 → 174,734, node visibility 3,413 → 11,104/11,421, flat across every depth band. Idempotent — a re-run fills 0 and leaves tree_revision alone (the ~60 MB tree payload should not be invalidated for a no-op). Residual ~27.6k rows have no named twin and need variant-coord-lift, which requires the reverse hs1->GRCh38 chain staged to establish its pivot. Co-Authored-By: Claude Opus 5 (1M context) --- rust/scripts/fill-y-tree-build-coords.sql | 139 ++++++++++++++++++++++ 1 file changed, 139 insertions(+) create mode 100644 rust/scripts/fill-y-tree-build-coords.sql diff --git a/rust/scripts/fill-y-tree-build-coords.sql b/rust/scripts/fill-y-tree-build-coords.sql new file mode 100644 index 00000000..76a330ef --- /dev/null +++ b/rust/scripts/fill-y-tree-build-coords.sql @@ -0,0 +1,139 @@ +-- One-off: fill missing GRCh38 (and GRCh37) coordinates on Y **tree-linked** variant rows by +-- copying them from the same marker's catalog row. +-- +-- WHY. The de-novo loader reuses a catalog row only when it matches on `coordinates @> {'hs1': …}`. +-- Markers whose hs1 coordinate had not been lifted yet missed that match, so the loader minted a +-- fresh hs1-only row — and *that* row is what `tree.haplogroup_variant` points at. The marker's +-- real coordinates stayed on the unlinked catalog row. `variant-name-reconcile` later adopted the +-- name onto the branch row but never touches `coordinates`, which is why the tree has correct +-- names and, on the 2026-08-04 prod dump, GRCh38 for only 44,181 of 203,983 branch SNPs (21.7%). +-- +-- WHY THIS MATTERS. The Navigator places each source in its *native* build with no liftover +-- (`place_y_consensus_decodingus`), and `parse_decodingus_json(json, build_key)` drops every locus +-- lacking that build's coordinate. So a GRCh38 subject saw only 3,413 of 11,421 Y nodes. All 33 +-- backbone nodes survived — what vanished was the terminal tree (21% node visibility at depth +-- 31-40), so GRCh38 subjects placed plausibly but SHALLOW rather than failing outright. +-- +-- WHY COPY RATHER THAN CHAIN-LIFT. The catalog row carries YBrowse's own GRCh38/GRCh37 values. +-- A chain lift would re-derive them and can mismap in the inverted / ampliconic Y blocks. On the +-- prod dump every one of the 132,183 candidate rows agreed with its twin on hs1 position AND +-- alleles (0 swapped, 0 other) and 132,181 had exactly one twin — so this copy is exact, not a +-- best guess. The match is enforced in the join below: a row whose twin disagrees on the hs1 site +-- simply is not filled. There is no unsafe write available to this script. +-- +-- SCOPE. Only rows linked into the *current* Y tree. mtDNA is untouched (the mt tree is +-- CP068254.1/hs1-native by design and carries no GRCh38 — that is not a defect). Rows with no +-- named twin are left alone and reported as the residual; they need `variant-coord-lift`, which +-- needs the REVERSE hs1->GRCh38 chain staged to establish its pivot (prod stages only the forward +-- hg38ToHs1 — without the reverse chain that job finishes clean with everything `no_source` and +-- looks like it worked). +-- +-- Idempotent: a filled row no longer matches `NOT coordinates ? 'GRCh38'`, so a re-run fills 0. +-- +-- Run: +-- PGPASSWORD=… psql -h localhost -U decoding_us_user -d decodingus_db \ +-- -v ON_ERROR_STOP=1 -f scripts/fill-y-tree-build-coords.sql +-- +-- AFTER (both matter): +-- decodingus-jobs run-once variant-representatives -- newly-shared builds let twins collapse +-- psql … -c 'ANALYZE core.variant;' +-- This script bumps tree.tree_revision itself — without that bump the Navigator keeps serving its +-- cached tree and none of this reaches a client. + +\set ON_ERROR_STOP on +\timing on + +BEGIN; + +\echo '--- before: Y tree-linked variants by build ---' +SELECT count(*) AS tree_variants, + count(*) FILTER (WHERE v.coordinates ? 'hs1') AS hs1, + count(*) FILTER (WHERE v.coordinates ? 'GRCh38') AS grch38, + count(*) FILTER (WHERE v.coordinates ? 'GRCh37') AS grch37 +FROM tree.haplogroup_variant hv +JOIN core.variant v ON v.id = hv.variant_id +JOIN tree.haplogroup h ON h.id = hv.haplogroup_id +WHERE hv.valid_until IS NULL AND h.valid_until IS NULL AND h.haplogroup_type = 'Y_DNA'; + +-- Candidate set: one twin per row. `DISTINCT ON` + the ORDER BY prefers the catalog +-- representative, then the lowest id, so the choice is deterministic across runs. +CREATE TEMP TABLE twin_fill ON COMMIT DROP AS +SELECT DISTINCT ON (v.id) v.id, o.id AS twin_id, o.coordinates AS src +FROM core.variant v +JOIN tree.haplogroup_variant hv ON hv.variant_id = v.id AND hv.valid_until IS NULL +JOIN tree.haplogroup h ON h.id = hv.haplogroup_id + AND h.haplogroup_type = 'Y_DNA' AND h.valid_until IS NULL +JOIN core.variant o + ON o.canonical_name = v.canonical_name + AND o.id <> v.id + AND o.coordinates ? 'GRCh38' + -- identical hs1 site AND alleles — this is what makes the copy exact rather than inferred + AND o.coordinates->'hs1'->>'contig' = v.coordinates->'hs1'->>'contig' + AND o.coordinates->'hs1'->>'position' = v.coordinates->'hs1'->>'position' + AND o.coordinates->'hs1'->>'ancestral' = v.coordinates->'hs1'->>'ancestral' + AND o.coordinates->'hs1'->>'derived' = v.coordinates->'hs1'->>'derived' +WHERE v.canonical_name IS NOT NULL + AND v.coordinates ? 'hs1' + AND NOT v.coordinates ? 'GRCh38' +ORDER BY v.id, o.catalog_representative DESC, o.id; + +\echo '--- rows this run will fill ---' +SELECT count(*) AS rows_to_fill FROM twin_fill; + +-- Fill GRCh38 always (that is the candidate predicate); ride GRCh37 along only where the row +-- lacks it and the twin has it. Existing keys are never overwritten. +UPDATE core.variant v +SET coordinates = v.coordinates + || jsonb_build_object('GRCh38', t.src->'GRCh38') + || CASE WHEN NOT v.coordinates ? 'GRCh37' AND t.src ? 'GRCh37' + THEN jsonb_build_object('GRCh37', t.src->'GRCh37') + ELSE '{}'::jsonb END, + updated_at = now() +FROM twin_fill t +WHERE v.id = t.id; + +\echo '--- after: Y tree-linked variants by build ---' +SELECT count(*) AS tree_variants, + count(*) FILTER (WHERE v.coordinates ? 'hs1') AS hs1, + count(*) FILTER (WHERE v.coordinates ? 'GRCh38') AS grch38, + count(*) FILTER (WHERE v.coordinates ? 'GRCh37') AS grch37 +FROM tree.haplogroup_variant hv +JOIN core.variant v ON v.id = hv.variant_id +JOIN tree.haplogroup h ON h.id = hv.haplogroup_id +WHERE hv.valid_until IS NULL AND h.valid_until IS NULL AND h.haplogroup_type = 'Y_DNA'; + +\echo '--- residual (no named twin — needs variant-coord-lift + the reverse chain) ---' +SELECT v.mutation_type::text AS mutation_type, + (v.canonical_name LIKE 'DU%') AS du_minted, + count(DISTINCT v.id) AS rows +FROM tree.haplogroup_variant hv +JOIN core.variant v ON v.id = hv.variant_id +JOIN tree.haplogroup h ON h.id = hv.haplogroup_id +WHERE hv.valid_until IS NULL AND h.valid_until IS NULL AND h.haplogroup_type = 'Y_DNA' + AND NOT v.coordinates ? 'GRCh38' +GROUP BY 1, 2 ORDER BY 3 DESC; + +\echo '--- node-level build visibility (what a subject on that build can actually see) ---' +WITH n AS ( + SELECT h.id, h.is_backbone, + count(*) FILTER (WHERE v.coordinates ? 'hs1') AS hs1, + count(*) FILTER (WHERE v.coordinates ? 'GRCh38') AS g38 + FROM tree.haplogroup h + JOIN tree.haplogroup_variant hv ON hv.haplogroup_id = h.id AND hv.valid_until IS NULL + JOIN core.variant v ON v.id = hv.variant_id + WHERE h.haplogroup_type = 'Y_DNA' AND h.valid_until IS NULL + GROUP BY 1, 2) +SELECT count(*) AS nodes_with_variants, + count(*) FILTER (WHERE hs1 > 0) AS visible_hs1, + count(*) FILTER (WHERE g38 > 0) AS visible_grch38, + count(*) FILTER (WHERE is_backbone AND g38 > 0) AS backbone_visible_grch38 +FROM n; + +-- Invalidate the served tree's ETag so clients re-fetch (tree endpoints answer 304 off this). +-- Only when this run actually changed something: the tree payload is ~60 MB, so a no-op bump +-- would make every client re-download it for nothing. +UPDATE tree.tree_revision SET revision = revision + 1, updated_at = now() +WHERE id = 1 AND EXISTS (SELECT 1 FROM twin_fill); +SELECT revision AS new_tree_revision FROM tree.tree_revision WHERE id = 1; + +COMMIT; From cd13e268b97ca832391163fe1652105966aa2140 Mon Sep 17 00:00:00 2001 From: James Kane Date: Thu, 6 Aug 2026 10:23:29 -0500 Subject: [PATCH 03/10] feat(tree): ancestral-origin icicle for the genealogical era MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public tree answers where a clade sits phylogenetically and, in the "Geography & Time" panel, where its samples were collected. Neither answers what a surname project actually asks: as this branch splits, where do the lines go? ytree.net answers it by putting geography onto the phylogeny itself. This adds that view — the Big Tree's top-down icicle, containment carrying descent, with each band filled by where its men's most distant known ancestors came from. The AppView had nothing to draw it from. Its only locality datum is `core.specimen_donor.geocoord`, and of 9,642 placed Y samples 1,380 carry one — all of them ancient or academic. The 7,882 `cohort=bigy` D2C tips, which are the entire genealogical era, have 3 between them. So this also adds the substrate: a `com.decodingus.atmosphere.ancestralOrigin` lexicon, mirrored to `fed.ancestral_origin` (migration 0074) by the existing Jetstream consumer. MDKA as publishable data is not new policy — `biosample-identifier-dedup.md` already records it as "genealogical context, not PII". What is new is that the AppView now *enforces* that rather than asserting it. Five gates run at ingest and REJECT the record, never store-and-hide, because a row that exists is a row some future read path can leak: a single-token surname (particles allowed, so `van der Berg` survives and `Thomas Michael Kane` does not), `birthYear <= 1900`, country-only when no birth year establishes the ancestor is long dead, coordinates re-coarsened to ~1 km whatever the client sent, and a join key that is never rendered. The bulk-load exclusion in `import_kit_identifiers.rs` stands: origins enter only when a PDS publishes them, so this ships dark until the Navigator half lands. The two migration headers that say otherwise (`0012_fed_reporting` here, `0030_mdka` in Navigator) are deliberately NOT edited — both repos run `sqlx::migrate!`, which checksums applied migrations, so changing even a comment would fail every existing database. `proposals/ancestral-origin-icicle.md` §2 is the amendment of record and 0074's header points at it. The D4 assertion-store rail rejecting MDKA_IS is untouched: it governs assertions about a *living* subject, which is the distinction that keeps the two apart. Resolution runs through `core.biosample_identifier`, not the at-uri. Zero placed samples carry an at-uri — the tips were bulk-loaded, not federated — while 7,548 carry an FTDNA row from migrations 0059/0060, which exist precisely to match a re-published donor to its existing biosample. Both paths are unioned so at-uri works as federation grows. Three things came from measuring rather than reasoning: * Place normalization was validated against all 3,356 real MDKA strings, which caught what hand-picked cases missed. US ZIPs went unstripped, making dozens of singleton "admins" (`Va 24521`, `Wv 26801`) that are all one state; the country table was too short; and a parenthetical qualifier dropped the row entirely. 705 distinct admin strings fold to 457, with 0 unresolvable and 6 no-country out of 3,356. * A band spans its PARENT's TMRCA to its own, not its own `formed_ybp`. The obvious choice is wrong: the two are independent point estimates under no monotonicity constraint, agreeing on 898 of 10,252 edges while 4,243 (41%) have the child forming before its parent's split. Rendering the real tree put `R-A13318` at exactly its parent `R-S764`'s y. Parent-TMRCA → own-TMRCA has zero inversions, so containment holds by construction. * Nothing is dropped silently. A sample on a de-novo node used to contribute to nothing, so every band above it understated itself; it now climbs to the nearest named ancestor as sample tips already do. Branches with no origin beneath them are pruned — unpruned, R-S764 drew 175 bands across 7,944px to show 10 origins; pruned it is 37 in 768px — and the count, the placed total, and the samples with no published origin are all stated on the page. Colour is the validated 8-slot categorical palette, fixed order, never cycled; a ninth locality folds into a reserved neutral that also carries "no locality recorded", since an absence is not an identity. Both modes pass the checker; light mode's contrast warning is met by the always-on band labels and the table view. Tests: 14 normalizer, 6 ingest-gate, 18 layout. Suites green (du-db 52, du-jobs 46, du-web 93). Verified end to end against the dev DB with synthetic records on real FTDNA kits, which resolved to real placed nodes; those rows were removed. Co-Authored-By: Claude Opus 5 (1M context) --- .../proposals/ancestral-origin-icicle.md | 173 ++++ rust/crates/du-db/src/fed/ancestral_origin.rs | 74 ++ rust/crates/du-db/src/fed/mod.rs | 4 + rust/crates/du-db/src/lib.rs | 2 + rust/crates/du-db/src/origins.rs | 120 +++ rust/crates/du-db/src/place.rs | 541 +++++++++++ rust/crates/du-jobs/src/jetstream.rs | 213 ++++- rust/crates/du-web/assets/main.css | 83 ++ rust/crates/du-web/src/main.rs | 1 + rust/crates/du-web/src/origins_layout.rs | 844 ++++++++++++++++++ rust/crates/du-web/src/routes/tree.rs | 167 ++++ .../crates/du-web/templates/tree/origins.html | 176 ++++ rust/locales/en.txt | 23 + rust/locales/es.txt | 23 + rust/locales/fr.txt | 23 + rust/migrations/0074_ancestral_origin.sql | 69 ++ 16 files changed, 2535 insertions(+), 1 deletion(-) create mode 100644 documents/proposals/ancestral-origin-icicle.md create mode 100644 rust/crates/du-db/src/fed/ancestral_origin.rs create mode 100644 rust/crates/du-db/src/origins.rs create mode 100644 rust/crates/du-db/src/place.rs create mode 100644 rust/crates/du-web/src/origins_layout.rs create mode 100644 rust/crates/du-web/templates/tree/origins.html create mode 100644 rust/migrations/0074_ancestral_origin.sql diff --git a/documents/proposals/ancestral-origin-icicle.md b/documents/proposals/ancestral-origin-icicle.md new file mode 100644 index 00000000..10621fa7 --- /dev/null +++ b/documents/proposals/ancestral-origin-icicle.md @@ -0,0 +1,173 @@ +# Ancestral-origin locality icicle on the public Y tree + +**Status:** proposed (2026-08-06). **AppView half only** — the Navigator publisher is deferred by +the project owner, so this ships *dark*: schema, ingest, aggregate and view are built and tested +against seeded records, and the surface fills when a PDS starts publishing. +**Scope:** a new `com.decodingus.atmosphere.ancestralOrigin` lexicon, `fed.ancestral_origin` +(migration 0074), a jetstream arm, `du_db::origins`, and a server-rendered icicle at +`/ytree/node/:name/origins`. No change to `core.biosample`, no change to placement. +**Privacy posture:** unchanged from `biosample-identifier-dedup.md` — the corpus carries no +living-donor PII, and MDKA (surname / origin / birth-year of the earliest paternal-line ancestor) is +**genealogical context, not PII**. This design does not relax that; it *enforces* it (§2). + +## 1. Why + +The public Y tree renders as a cladogram (`du-web/src/tree_layout.rs`) with a per-clade Leaflet map +in the "Geography & Time" panel. Neither answers the question a surname project asks: **as this +branch splits, where do the lines go?** ytree.net answers it by putting geography onto the phylogeny +itself — depth down the page, a block's height its elapsed time, each block spanning its +descendants. This adds that view with locality as the fill. + +It only means anything in the genealogical era. Deeper than ~1,500 ybp every block aggregates to +"Europe" and the counts explode, so the view is age-gated rather than offered tree-wide (§5). + +**The data gap, measured on `decodingus_cutover` (2026-08-06).** The AppView's only locality datum +is `core.specimen_donor.geocoord`; there is no country and no place text, and legacy had none +either, so the ETL did not drop one. Of **9,642 placed Y samples, 1,380 carry a coordinate**, and +that coverage is entirely ancient/academic: + +| source | placed | with geocoord | +|---|---:|---:| +| `EXTERNAL` (ancient + academic) | 1,760 | 1,377 | +| `STANDARD` (`cohort=bigy` D2C tips) | 7,882 | **3** | + +The genealogical era — the only era this view is for — has effectively no locality data. That is +what the lexicon exists to supply. + +## 2. What may cross the wire, and how it is enforced + +The posture is already the project's (`biosample-identifier-dedup.md` §Privacy posture). What is new +is that this design **enforces** it at ingest rather than asserting it in prose. A record failing any +gate is **rejected**, not merely un-rendered: + +1. **Surname only.** No given name, ever. The publisher derives it; the AppView independently + rejects a `surname` containing whitespace or more than one name token, so a buggy or hostile + client cannot leak a given name through a field labelled `surname`. +2. **Date ceiling — `birthYear <= 1900`.** A person born in 1900 is 126 today. This is the check + that makes "not PII" verifiable rather than asserted. +3. **Precision ladder when the birth year is absent.** With a birth year: place text + coarsened + coordinate. Without one: **country only** — place text and coordinate are dropped at ingest. +4. **Coordinates coarsened to 2 decimal places (~1 km).** Applied at publish *and* re-applied at + ingest, because the client cannot be trusted to have done it. A county-scale view cannot use more + precision; full precision plus a surname narrows to one family. +5. **The join key is never rendered.** Resolution runs through an FTDNA kit id (§4), and every vendor + namespace is `is_public = false` (`du_db::identifier::is_public_namespace`). The icicle shows + `Kane · Co. Clare`; the kit number must not reach any public projection. + +**The bulk-load exclusion stands.** `du-jobs/src/import_kit_identifiers.rs:17` records the decision +that MDKA "enters only when a PDS publishes the sample, never from this bulk load." This design does +not create a manifest or curator path. It is the reason the view ships dark, and that is accepted. + +**Two migration headers say the opposite and are deliberately not edited.** Both repos use +`sqlx::migrate!`, which checksums applied migrations — editing a comment in +`0030_mdka.up.sql` (Navigator) or `0012_fed_reporting.sql` (AppView) would fail every existing +database with `VersionMismatch`. **This document is the amendment of record**, and migration 0074's +header points back to it. + +The D4 assertion store's PII rail (`research.assertion` rejecting `MDKA_IS`) **stands unchanged**. +It governs assertions made *about a living research subject* within a project, which is a different +question from publishing a deceased ancestor's parish — and the rail is what keeps the two apart. + +## 3. The record + +`com.decodingus.atmosphere.ancestralOrigin`, one per `(biosample, lineage)`: + +```jsonc +{ + "biosampleRef": "at://did:plc:…/com.decodingus.atmosphere.biosample/…", // when federated + "externalIds": [{ "namespace": "FTDNA", "value": "B5163" }], // the join that fires + "lineage": "Y_DNA", // Y_DNA | MT_DNA + "surname": "Kane", // single token; never a given name + "originPlace": "Creegh South, Co. Clare, Ireland", // as recorded; normalized server-side + "originCountry": "Ireland", + "birthYear": 1830, + "deathYear": 1908, + "lat": 52.75, // 2dp + "lon": -9.43, + "createdAt": "2026-08-06T…Z" +} +``` + +**Place text is published as recorded and normalized in the AppView**, not at the edge. One +implementation, fixable without a client release, and re-runnable over records already ingested. The +normalizer is a pure function in `du_db::place` with a country/admin synonym table — the corpus needs +it: `Ireland` / `Republic of Ireland` / `ireland`; `UK` / `United Kingdom` / `Scotland`; `Co. Cork` +vs `Cork`; `VA` vs `Virginia`; UK postcodes embedded mid-string (`Moulin, Pitlochry PH16 5EP, UK`). +705 distinct raw admin strings across the reference corpus. + +## 4. Resolving a record to a placed sample + +The obvious join — `core.biosample.atproto->>'uri' = biosample_ref`, as `discovery.rs:185` does — +**matches nothing**: zero placed samples carry an at-uri, because the tips were bulk-loaded rather +than federated. + +The working key already exists. **All 7,548 placed bigy tips carry an `FTDNA` row in +`core.biosample_identifier`** (migrations 0059/0060, built precisely to "match a re-published donor +to its existing biosample"). So resolution is `(namespace, value)` against that table, with the +at-uri as a fallback for genuinely federated samples. No re-federation, no new identity work. + +## 5. The view + +`/ytree/node/:name/origins` (+ the mt sibling), a server-rendered inline SVG — no client layout +library, matching `tree_layout.rs`. + +- **Geometry**: depth → y; each node a rect spanning its subtree's horizontal extent; children flush + against the parent's underside, so containment carries descent and no connector is drawn. +- **Height = elapsed years**: a branch spans **its parent's TMRCA → its own TMRCA**, on one absolute + calendar axis. This is the deliberate divergence from Navigator's SNP-count height, and the reason + to build the view here: the AppView has ages (`tmrca_ybp` on 10,257 of 11,422 Y nodes) and the + framing is temporal. Nodes with no age draw at a minimum height, hatched, and are excluded from + the ruler — visible, not silently normal. + + **Do not use a node's own `formed_ybp` for the top of its band.** It is the obvious choice and it + is wrong: `formed_ybp` and the parent's `tmrca_ybp` are independent point estimates under no + monotonicity constraint, and on the live tree they agree on only **898 of 10,252 edges** while + **4,243 (41%) have the child forming earlier than its parent's split**. Driving geometry from it + draws children on top of their parents — caught by rendering the real tree, where `R-A13318` + (formed 1622) landed at exactly its parent `R-S764`'s y. Parent-TMRCA → own-TMRCA has **zero** + inversions over the same edges, so containment holds by construction. +- **Fill = stacked locality composition** of the placed samples at or below the block, at the + selected level (Country / Admin1 / Place), with **"no locality recorded" always its own visible + slice**. A view of who published is not a view of where a branch is from, and the difference must + be on screen. +- **Tips**: one leaf box per placed sample carrying an origin — `Kane · Co. Clare`, coloured to + match. Never the kit id. +- **Colours**: categorical, colourblind-safe, legible in both themes; assigned by frequency rank + *within the rendered subtree* (deterministic, tie-broken on name), top N distinct + a neutral + "other". +- **Era gate**: serves nodes with `tmrca_ybp <= 1500` (adjustable within bounds). Above the cutoff it + renders the breadcrumb, one line of explanation, and links down to eligible children rather than + drawing a block that means nothing. +- **Pruned to the branches that carry an origin**, with the count reported. A branch with no + published origin beneath it is a column of width and no information: on `R-S764` the unpruned + draw was 175 bands across a 7,944px canvas to show 10 origins; pruned it is 37 bands in 768px. + The drawn depth therefore follows the data rather than a fixed window — which also means no + sample is lost for sitting below a cut-off. +- **De-novo nodes stay hidden but their men still count.** A sample placed on an auto-named node is + attributed to the nearest named ancestor, as the public tree already does for sample tips. + Dropping it instead made every band above it understate its own composition. +- **No silent caps.** Pruned branches, samples with no published origin, and the placed total are + all stated on the page. + +## 6. Phasing + +1. `du_db::place` normalizer — pure, unit-tested. *(No wire format, nothing published.)* +2. Migration 0074 + `fed::ancestral_origin` + the jetstream arm + every gate in §2. +3. `du_db::origins` aggregate + `origins_layout` + route + template + i18n. +4. **Deferred, Navigator:** the lexicon's client half — `AncestralOriginRecord`, the surname + splitter, and the publish predicate (the workspace holds primary data for the subject **and** + `ftdna_member.publicly_shares = 1` or there is no roster row). Measured on the reference + workspace: 583 Y MDKA rows sit on subjects with primary data, 558 of them publicly-sharing — + **the 25 that are not must never publish.** + +## 7. Open items + +- **The precision ladder (§2.3)** is a proposed default, not a derived rule. It withholds place-level + detail for the ~57% of MDKA rows with no birth year. +- **The 1900 ceiling** is a round number, not a legal standard. Cheap to set now, expensive to lower + once records exist. +- **Retraction.** A withdrawn consent needs the PDS record deleted *and* the mirror tombstoned. The + jetstream `delete` path (`jetstream.rs:163`) is the mechanism; the workflow is unspecified. +- **The view ships empty** until the Navigator half lands. That is a consequence of §2's bulk-load + exclusion, not a defect. +- **mtDNA** costs almost nothing extra (the lexicon is lineage-keyed). Ship Y first and validate there. diff --git a/rust/crates/du-db/src/fed/ancestral_origin.rs b/rust/crates/du-db/src/fed/ancestral_origin.rs new file mode 100644 index 00000000..0cb8a7b8 --- /dev/null +++ b/rust/crates/du-db/src/fed/ancestral_origin.rs @@ -0,0 +1,74 @@ +//! Mirrored ancestral-origin records (`com.decodingus.atmosphere.ancestralOrigin`) — one +//! lineage's most distant known ancestor: surname, origin, dates. The locality substrate for the +//! genealogical-era origins icicle. See [`super`] for the shared cursor/delete, and +//! `proposals/ancestral-origin-icicle.md` for the design. +//! +//! **This layer is pure storage.** The privacy gates that make these records publishable — +//! single-token surname, `birth_year <= 1900`, country-only without a birth year, coordinates +//! coarsened — run in the consumer (`du_jobs::jetstream::build_ancestral_origin`) *before* a row +//! reaches here, matching how every other `fed.*` module leaves record-shape extraction to +//! du-jobs. A row in this table has already passed them. + +use super::Common; +use crate::DbError; +use serde_json::Value; +use sqlx::PgPool; + +/// A mirrored ancestral origin, post-gate. Every field beyond the envelope is optional because +/// the precision ladder legitimately produces a country-only record. +pub struct AncestralOrigin { + pub common: Common, + /// at-uri of the parent biosample record — present only for genuinely federated samples. + pub biosample_ref: Option, + /// `[{namespace, value}]` verbatim — the join key that actually fires for tree tips. + pub external_ids: Value, + pub lineage: Option, + pub surname: Option, + pub origin_place: Option, + pub origin_country: Option, + pub birth_year: Option, + pub death_year: Option, + /// Coarsened to 2dp by the consumer. `None` when the record carried no usable coordinate. + pub lat: Option, + pub lon: Option, +} + +pub async fn upsert(pool: &PgPool, o: &AncestralOrigin) -> Result<(), DbError> { + sqlx::query( + "INSERT INTO fed.ancestral_origin \ + (did, rkey, at_uri, cid, biosample_ref, external_ids, lineage, surname, \ + origin_place, origin_country, birth_year, death_year, geocoord, record_created_at, time_us) \ + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12, \ + CASE WHEN $13::float8 IS NULL OR $14::float8 IS NULL THEN NULL \ + ELSE ST_SetSRID(ST_MakePoint($14::float8, $13::float8), 4326) END, \ + $15,$16) \ + ON CONFLICT (did, rkey) DO UPDATE SET \ + at_uri = EXCLUDED.at_uri, cid = EXCLUDED.cid, biosample_ref = EXCLUDED.biosample_ref, \ + external_ids = EXCLUDED.external_ids, lineage = EXCLUDED.lineage, \ + surname = EXCLUDED.surname, origin_place = EXCLUDED.origin_place, \ + origin_country = EXCLUDED.origin_country, birth_year = EXCLUDED.birth_year, \ + death_year = EXCLUDED.death_year, geocoord = EXCLUDED.geocoord, \ + record_created_at = EXCLUDED.record_created_at, time_us = EXCLUDED.time_us, \ + indexed_at = now() \ + WHERE EXCLUDED.time_us >= fed.ancestral_origin.time_us", + ) + .bind(&o.common.did) + .bind(&o.common.rkey) + .bind(&o.common.at_uri) + .bind(&o.common.cid) + .bind(&o.biosample_ref) + .bind(&o.external_ids) + .bind(&o.lineage) + .bind(&o.surname) + .bind(&o.origin_place) + .bind(&o.origin_country) + .bind(o.birth_year) + .bind(o.death_year) + .bind(o.lat) + .bind(o.lon) + .bind(o.common.record_created_at) + .bind(o.common.time_us) + .execute(pool) + .await?; + Ok(()) +} diff --git a/rust/crates/du-db/src/fed/mod.rs b/rust/crates/du-db/src/fed/mod.rs index 81456e6f..344cfa4c 100644 --- a/rust/crates/du-db/src/fed/mod.rs +++ b/rust/crates/du-db/src/fed/mod.rs @@ -18,6 +18,7 @@ use chrono::{DateTime, Utc}; use sqlx::PgPool; pub mod analytics; +pub mod ancestral_origin; pub mod core; pub mod coverage; pub mod device_key; @@ -45,6 +46,7 @@ pub const NS_INSTRUMENT_OBSERVATION: &str = "com.decodingus.atmosphere.instrumen pub const NS_PRIVATE_VARIANT: &str = "com.decodingus.atmosphere.privateVariant"; pub const NS_DEVICE_KEY: &str = "com.decodingus.atmosphere.deviceKey"; pub const NS_FEED_POST: &str = "com.decodingus.atmosphere.feed.post"; +pub const NS_ANCESTRAL_ORIGIN: &str = "com.decodingus.atmosphere.ancestralOrigin"; /// Every collection mirrored for reporting (the consumer's `wantedCollections`). pub const INGEST_COLLECTIONS: &[&str] = &[ @@ -61,6 +63,7 @@ pub const INGEST_COLLECTIONS: &[&str] = &[ NS_PRIVATE_VARIANT, NS_DEVICE_KEY, NS_FEED_POST, + NS_ANCESTRAL_ORIGIN, ]; /// The `fed.*` reporting table backing a collection, or `None` if unsupported. @@ -79,6 +82,7 @@ fn table_for(collection: &str) -> Option<&'static str> { NS_PRIVATE_VARIANT => "fed.private_variant", NS_DEVICE_KEY => "fed.device_key", NS_FEED_POST => "fed.feed_post", + NS_ANCESTRAL_ORIGIN => "fed.ancestral_origin", _ => return None, }) } diff --git a/rust/crates/du-db/src/lib.rs b/rust/crates/du-db/src/lib.rs index 22081bf9..d865de4c 100644 --- a/rust/crates/du-db/src/lib.rs +++ b/rust/crates/du-db/src/lib.rs @@ -31,8 +31,10 @@ pub mod job_lock; pub mod merge; pub mod naming; pub mod notification; +pub mod origins; pub mod pagination; pub mod pdf; +pub mod place; pub mod proposal; pub mod publication; pub mod recruitment; diff --git a/rust/crates/du-db/src/origins.rs b/rust/crates/du-db/src/origins.rs new file mode 100644 index 00000000..7dd93bb3 --- /dev/null +++ b/rust/crates/du-db/src/origins.rs @@ -0,0 +1,120 @@ +//! Ancestral origins of the placed samples under a clade — the read side of +//! `fed.ancestral_origin`, and the input to the genealogical-era origins icicle. +//! Design: `proposals/ancestral-origin-icicle.md`. +//! +//! **Two resolution paths, because the tree predates federation.** A published origin names its +//! sample either by the parent biosample's at-uri (genuinely federated samples) or by a vendor +//! identifier. The second is the one that fires: **no** placed sample currently carries an at-uri +//! — the tips were bulk-loaded — while 7,548 of them carry an `FTDNA` row in +//! `core.biosample_identifier`. Both are unioned so the at-uri path works as federation grows. +//! +//! **The identifier never leaves this module.** It is a vendor kit id, `is_public = false` by +//! namespace policy ([`crate::identifier::is_public_namespace`]); it is a join key here and must +//! not reach any public projection. [`SampleOrigin`] deliberately carries no identifier field. + +use crate::place::{self, PlacePath}; +use crate::{pg_enum_label, DbError}; +use du_domain::enums::DnaType; +use sqlx::PgPool; +use uuid::Uuid; + +/// One placed sample's ancestral origin, normalized and ready to group by. +#[derive(Debug, Clone)] +pub struct SampleOrigin { + pub sample_guid: Uuid, + /// The node this sample is placed on — the composition rolls up from here to every ancestor. + pub haplogroup_id: i64, + /// Family name of the most distant known ancestor. Single-token by ingest gate. + pub surname: Option, + /// Normalized locality ladder. May be empty when the record carried no place at all. + pub place: PlacePath, + /// The ancestor's birth year — also the reason place-level detail was allowed to publish. + pub birth_year: Option, +} + +/// Published origins for every placed sample at or below `root_name`. +/// +/// The subtree is walked **unbounded**, not to the render window's depth: a block's composition is +/// what lies beneath it, and cutting the walk at the visible depth would silently understate every +/// block at the boundary. +/// +/// When two contributors publish an origin for the same sample the most recently indexed record +/// wins (`time_us`), so the result holds at most one row per sample and the counts cannot +/// double-count a man. +pub async fn origins_under( + pool: &PgPool, + dna_type: DnaType, + root_name: &str, +) -> Result, DbError> { + #[derive(sqlx::FromRow)] + struct Row { + sample_guid: Uuid, + haplogroup_id: i64, + surname: Option, + origin_place: Option, + origin_country: Option, + birth_year: Option, + } + let rows: Vec = sqlx::query_as( + "WITH RECURSIVE sub AS ( \ + SELECT id FROM tree.haplogroup \ + WHERE name = $1 AND haplogroup_type::text = $2 AND valid_until IS NULL \ + UNION ALL \ + SELECT r.child_haplogroup_id FROM tree.haplogroup_relationship r \ + JOIN sub ON r.parent_haplogroup_id = sub.id \ + WHERE r.valid_until IS NULL \ + ), \ + placed AS ( \ + SELECT hs.sample_guid, hs.haplogroup_id, b.atproto->>'uri' AS at_uri \ + FROM tree.haplogroup_sample hs \ + JOIN sub ON sub.id = hs.haplogroup_id \ + JOIN core.biosample b ON b.sample_guid = hs.sample_guid AND b.deleted = false \ + WHERE hs.dna_type::text = $2 AND hs.status IN ('PLACED','CURATED') \ + ), \ + ids AS ( \ + SELECT ao.did, ao.rkey, upper(e->>'namespace') AS ns, upper(e->>'value') AS val \ + FROM fed.ancestral_origin ao, LATERAL jsonb_array_elements(ao.external_ids) e \ + WHERE ao.lineage = $2 \ + ), \ + matched AS ( \ + SELECT p.sample_guid, p.haplogroup_id, ao.surname, ao.origin_place, \ + ao.origin_country, ao.birth_year, ao.time_us \ + FROM placed p \ + JOIN core.biosample_identifier i ON i.sample_guid = p.sample_guid \ + JOIN ids ON ids.ns = i.namespace AND ids.val = i.value \ + JOIN fed.ancestral_origin ao ON ao.did = ids.did AND ao.rkey = ids.rkey \ + UNION ALL \ + SELECT p.sample_guid, p.haplogroup_id, ao.surname, ao.origin_place, \ + ao.origin_country, ao.birth_year, ao.time_us \ + FROM placed p \ + JOIN fed.ancestral_origin ao ON ao.biosample_ref = p.at_uri \ + WHERE p.at_uri IS NOT NULL AND ao.lineage = $2 \ + ) \ + SELECT DISTINCT ON (sample_guid) \ + sample_guid, haplogroup_id, surname, origin_place, origin_country, birth_year \ + FROM matched \ + ORDER BY sample_guid, time_us DESC", + ) + .bind(root_name) + .bind(pg_enum_label(&dna_type)?) + .fetch_all(pool) + .await?; + + Ok(rows + .into_iter() + .map(|r| SampleOrigin { + sample_guid: r.sample_guid, + haplogroup_id: r.haplogroup_id, + surname: r.surname, + place: place::normalize(r.origin_place.as_deref(), r.origin_country.as_deref()), + birth_year: r.birth_year, + }) + .collect()) +} + +/// How many placed samples sit at or below `root_name` — the denominator the view reports +/// alongside the composition, so "12 origins" is never mistaken for "12 men". +/// +/// [`crate::tree_sample::count_under`] answers the same question and is reused rather than +/// duplicated; this alias exists only to keep the origins call site reading in one place. +pub use crate::tree_sample::count_under as placed_under; diff --git a/rust/crates/du-db/src/place.rs b/rust/crates/du-db/src/place.rs new file mode 100644 index 00000000..1fc03ff5 --- /dev/null +++ b/rust/crates/du-db/src/place.rs @@ -0,0 +1,541 @@ +//! Locality normalization for published ancestral origins — a pure function turning the +//! free-text place a client recorded into a `country / admin / locality` ladder the icicle can +//! group by. See `proposals/ancestral-origin-icicle.md` §3. +//! +//! **Why this lives server-side.** The wire record carries the place *as recorded*, because one +//! normalizer in the AppView is fixable without a client release and can be re-run over records +//! already ingested. A normalizer at the edge would freeze whatever each client shipped with. +//! +//! **Why it is heuristic, and stays heuristic.** The input is geocoder output — comma-separated, +//! country last — not a gazetteer key. Two synonym tables cover what the corpus actually needs +//! (Irish counties, US states); everything else passes through as recorded rather than being +//! guessed at. A wrong fold is worse than an unfolded label: it silently merges two branches' +//! origins into one slice of a chart. +//! +//! ```text +//! "Raheen, Clashmore, Co. Waterford, Ireland" → Ireland / Co. Waterford / Raheen +//! "Pickens County, SC, USA" → United States / South Carolina / Pickens County +//! "Moulin, Pitlochry PH16 5EP, UK" → United Kingdom / Pitlochry / Moulin +//! "Ireland" → Ireland / — / — +//! ``` + +/// A normalized place, coarsest first. Every field is independently optional: a record may carry +/// a country and nothing else, which is exactly what the §2.3 precision ladder produces for an +/// origin with no ancestor birth year. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct PlacePath { + /// Canonical country. The four UK constituent countries stay distinct from `United Kingdom` + /// — `Scotland` vs `England` is the distinction a Y project reads by, and folding them to + /// `United Kingdom` would destroy it. + pub country: Option, + /// Canonical first-level division: an Irish county (`Co. Cork`), a US state (`Virginia`), or + /// whatever was recorded where no synonym table applies. + pub admin: Option, + /// The finest named place recorded below `admin`. + pub locality: Option, +} + +/// Which rung of the ladder a view groups by. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Level { + Country, + Admin, + Locality, +} + +impl PlacePath { + /// The label at `level`, falling back **coarser** when the requested rung is absent — a + /// sample known only to `Ireland` groups under `Ireland` at every level rather than + /// disappearing into "unknown", which would understate what the branch does tell us. + /// + /// `None` means no locality at all was recorded, and the view must draw that as its own + /// visible slice. + pub fn label_at(&self, level: Level) -> Option<&str> { + let ladder: [&Option; 3] = match level { + Level::Country => [&self.country, &None, &None], + Level::Admin => [&self.admin, &self.country, &None], + Level::Locality => [&self.locality, &self.admin, &self.country], + }; + ladder.into_iter().flatten().next().map(String::as_str) + } + + pub fn is_empty(&self) -> bool { + self.country.is_none() && self.admin.is_none() && self.locality.is_none() + } +} + +/// Normalize a recorded place string and/or a separately recorded country into a [`PlacePath`]. +/// +/// `place` is geocoder-shaped (`"Cork, Co. Cork, Ireland"`); `country` is whatever the client +/// recorded in its own country field, used when `place` is absent or carries no recognizable +/// country. Both may be absent, which yields an empty path. +pub fn normalize(place: Option<&str>, country: Option<&str>) -> PlacePath { + let parts: Vec = place + .unwrap_or_default() + .split(',') + .map(strip_postcode) + .filter(|s| !s.is_empty()) + .collect(); + + // Consume the trailing country component(s). `"Chelmsford, England, UK"` spends two: a bare + // `UK` behind a constituent country is a geocoder artefact, and the constituent country is + // the more informative of the two. + let mut rest: &[String] = &parts; + let mut resolved = None; + if let Some((last, head)) = rest.split_last() { + if let Some(c) = canonical_country(last) { + resolved = Some(c); + rest = head; + if resolved.as_deref() == Some(UNITED_KINGDOM) { + if let Some((prev, prev_head)) = rest.split_last() { + if let Some(inner) = canonical_country(prev) { + if inner != UNITED_KINGDOM { + resolved = Some(inner); + rest = prev_head; + } + } + } + } + } + } + // Still no country, but the string ends in a US state (`"Blount Co., AL"`). Infer it: a bare + // state token in the trailing position is a US address. This runs *after* country matching, + // so the codes that collide with countries are already claimed — `CA` is Canada, `DE` is + // Germany, `IN` is India — and only genuinely unclaimed state tokens reach here. + // The state itself stays in `rest`: it is this string's admin component, and whatever sits + // above it is still the locality (`"Blount Co., AL"` → Alabama / Blount Co.). + if resolved.is_none() && rest.last().is_some_and(|l| us_state(l).is_some()) { + resolved = Some("United States".to_string()); + } + // No country anywhere in the place string — fall back to the recorded country field, which is + // *declared* to be a country, so an unrecognized value is taken at its word rather than + // dropped. The strict table is only needed for the place string, where recognition is how we + // tell a country component from a place component; here there is nothing to disambiguate. + // Without this, every origin outside the synonym table (Israel, Cuba, Isle of Man, Guernsey…) + // vanished into "no locality recorded" despite having one. + let country = resolved.or_else(|| { + country + .map(|c| canonical_country(c).unwrap_or_else(|| titled(strip_qualifier(c)))) + .filter(|c| !c.is_empty()) + }); + + let admin = rest + .last() + .and_then(|raw| canonical_admin(country.as_deref(), raw)); + // Only when something sits *above* the admin component is there a finer locality to name. + let locality = (rest.len() >= 2).then(|| titled(&rest[0])); + + PlacePath { + country, + admin, + locality, + } +} + +const UNITED_KINGDOM: &str = "United Kingdom"; + +/// Fold a country token to its canonical name, or `None` when it is not a country at all — which +/// is how [`normalize`] tells a country component from a place component. +/// +/// The table covers what the corpus needs; it is deliberately not exhaustive, because a *declared* +/// country field is trusted as-is by [`normalize`] rather than being checked against this list. +pub fn canonical_country(raw: &str) -> Option { + let key = squash(strip_qualifier(raw)); + let name = match key.as_str() { + "ireland" | "republic of ireland" | "eire" | "ie" | "irl" => "Ireland", + "northern ireland" | "n ireland" | "n. ireland" | "ulster" => "Northern Ireland", + "scotland" | "alba" => "Scotland", + "england" => "England", + "wales" | "cymru" => "Wales", + "uk" | "u.k." | "united kingdom" | "great britain" | "britain" | "gb" => UNITED_KINGDOM, + "usa" | "u.s.a." | "us" | "u.s." | "united states" | "united states of america" => "United States", + "canada" | "ca" => "Canada", + "australia" | "au" => "Australia", + "new zealand" | "nz" => "New Zealand", + "germany" | "deutschland" | "de" => "Germany", + "france" | "fr" => "France", + "spain" | "espana" | "es" => "Spain", + "italy" | "italia" | "it" => "Italy", + "norway" | "norge" | "no" => "Norway", + "sweden" | "sverige" | "se" => "Sweden", + "denmark" | "danmark" | "dk" => "Denmark", + "netherlands" | "the netherlands" | "holland" | "nl" => "Netherlands", + "belgium" | "be" => "Belgium", + "switzerland" | "ch" => "Switzerland", + "austria" | "at" => "Austria", + "poland" | "polska" | "pl" => "Poland", + "portugal" | "pt" => "Portugal", + "finland" | "suomi" | "fi" => "Finland", + "russia" | "russian federation" | "ru" => "Russia", + "iceland" | "is" => "Iceland", + "luxembourg" | "lu" => "Luxembourg", + "czech republic" | "czechia" | "cz" => "Czech Republic", + "hungary" | "hu" => "Hungary", + "greece" | "gr" => "Greece", + "turkey" | "turkiye" | "tr" => "Turkey", + "ukraine" | "ua" => "Ukraine", + "romania" | "ro" => "Romania", + "india" | "in" => "India", + "china" | "cn" => "China", + "japan" | "jp" => "Japan", + "mexico" | "mx" => "Mexico", + "brazil" | "br" => "Brazil", + "argentina" | "ar" => "Argentina", + "south africa" | "za" => "South Africa", + // Caribbean and Atlantic origins recur in the corpus alongside the Irish diaspora. + "barbados" | "bb" => "Barbados", + "jamaica" | "jm" => "Jamaica", + "aruba" | "aw" => "Aruba", + "martinique" | "mq" => "Martinique", + "cayman islands" | "ky" => "Cayman Islands", + "saint kitts and nevis" | "st kitts and nevis" | "kn" => "Saint Kitts and Nevis", + "bermuda" | "bm" => "Bermuda", + _ => return None, + }; + Some(name.to_string()) +} + +/// Fold a first-level division against its country's synonym table. Unknown values pass through +/// title-cased rather than being dropped — an unrecognized county is still a real distinction. +fn canonical_admin(country: Option<&str>, raw: &str) -> Option { + let key = squash(raw); + if key.is_empty() { + return None; + } + match country { + // The 32 counties, however the geocoder spelled them. `Cork` and `Co. Cork` are the same + // county; `Cork` the city normalizes here too, and is recovered as the locality when the + // string carried one. + Some("Ireland") | Some("Northern Ireland") => { + let bare = key + .trim_start_matches("county ") + .trim_start_matches("co. ") + .trim_start_matches("co ") + .trim(); + IRISH_COUNTIES + .iter() + .find(|c| squash(c) == bare) + .map(|c| format!("Co. {c}")) + .or_else(|| Some(titled(raw))) + } + Some("United States") => us_state(raw).or_else(|| Some(titled(raw))), + _ => Some(titled(raw)), + } +} + +/// A US state by postal abbreviation or full name, canonicalized to the full name. `None` for +/// anything else — which is also how [`normalize`] recognizes a bare trailing state token. +fn us_state(raw: &str) -> Option { + let key = squash(raw); + US_STATES + .iter() + .find(|(abbr, name)| key == squash(abbr) || key == squash(name)) + .map(|(_, name)| (*name).to_string()) +} + +/// Drop a parenthetical qualifier: the corpus writes `"United States (Native American)"`, which is +/// an ancestry note attached to a country field, and keeping it would put that lineage in a +/// country of its own. +fn strip_qualifier(raw: &str) -> &str { + match raw.split_once('(') { + Some((head, _)) if !head.trim().is_empty() => head, + _ => raw, + } +} + +/// Lowercase, collapse internal whitespace, drop surrounding punctuation — the comparison key. +fn squash(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut space = false; + for ch in s.trim().chars() { + if ch.is_whitespace() { + space = !out.is_empty(); + } else { + if space { + out.push(' '); + space = false; + } + out.extend(ch.to_lowercase()); + } + } + out +} + +/// Trim a component and remove an embedded postal code. Geocoded strings carry them mid-string — +/// `"Pitlochry PH16 5EP"`, `"VA 24521"` — where they would otherwise make every town and every +/// ZIP its own distinct admin. Measured on the reference corpus, leaving US ZIPs in produced +/// dozens of singleton "admins" (`Va 24521`, `Wv 26801`) that are all one state. +fn strip_postcode(part: &str) -> String { + let kept: Vec<&str> = part + .split_whitespace() + .filter(|t| !is_uk_postcode_token(t) && !is_us_zip_token(t)) + .collect(); + kept.join(" ").trim().to_string() +} + +/// A UK postcode half: an outward code (`PH16`, `SW1A`) or an inward code (`5EP`) — a token mixing +/// letters and digits, no longer than four characters. Deliberately narrow: `"1st"` and ordinary +/// words must survive, and a real place name never looks like this. +fn is_uk_postcode_token(t: &str) -> bool { + let t = t.trim_matches(|c: char| !c.is_alphanumeric()); + (2..=4).contains(&t.len()) + && t.chars().all(|c| c.is_ascii_alphanumeric()) + && t.chars().any(|c| c.is_ascii_digit()) + && t.chars().any(|c| c.is_ascii_alphabetic()) +} + +/// A US ZIP (`24521`) or ZIP+4 (`22554-7232`). Five digits exactly, so four-digit years — which +/// the corpus does carry in free-text notes — survive. +fn is_us_zip_token(t: &str) -> bool { + let t = t.trim_matches(|c: char| !c.is_alphanumeric()); + let (head, tail) = match t.split_once('-') { + Some((h, t4)) => (h, Some(t4)), + None => (t, None), + }; + head.len() == 5 + && head.chars().all(|c| c.is_ascii_digit()) + && tail.is_none_or(|t4| t4.len() == 4 && t4.chars().all(|c| c.is_ascii_digit())) +} + +/// Title-case a pass-through label, preserving what the recorder wrote where it is already mixed +/// case (`"Na h-Eileanan an Iar"`, `"O'Brien"`) — only an all-lower or all-upper token is recased. +fn titled(raw: &str) -> String { + let s = raw.trim(); + let uniform = s.chars().filter(|c| c.is_alphabetic()).all(char::is_lowercase) + || s.chars().filter(|c| c.is_alphabetic()).all(char::is_uppercase); + if !uniform { + return s.to_string(); + } + s.split(' ') + .map(|w| { + let mut chars = w.chars(); + match chars.next() { + Some(f) => f.to_uppercase().collect::() + &chars.as_str().to_lowercase(), + None => String::new(), + } + }) + .collect::>() + .join(" ") +} + +const IRISH_COUNTIES: [&str; 32] = [ + "Antrim", "Armagh", "Carlow", "Cavan", "Clare", "Cork", "Derry", "Donegal", "Down", "Dublin", + "Fermanagh", "Galway", "Kerry", "Kildare", "Kilkenny", "Laois", "Leitrim", "Limerick", + "Longford", "Louth", "Mayo", "Meath", "Monaghan", "Offaly", "Roscommon", "Sligo", "Tipperary", + "Tyrone", "Waterford", "Westmeath", "Wexford", "Wicklow", +]; + +const US_STATES: [(&str, &str); 51] = [ + ("AL", "Alabama"), ("AK", "Alaska"), ("AZ", "Arizona"), ("AR", "Arkansas"), + ("CA", "California"), ("CO", "Colorado"), ("CT", "Connecticut"), ("DE", "Delaware"), + ("DC", "District of Columbia"), ("FL", "Florida"), ("GA", "Georgia"), ("HI", "Hawaii"), + ("ID", "Idaho"), ("IL", "Illinois"), ("IN", "Indiana"), ("IA", "Iowa"), ("KS", "Kansas"), + ("KY", "Kentucky"), ("LA", "Louisiana"), ("ME", "Maine"), ("MD", "Maryland"), + ("MA", "Massachusetts"), ("MI", "Michigan"), ("MN", "Minnesota"), ("MS", "Mississippi"), + ("MO", "Missouri"), ("MT", "Montana"), ("NE", "Nebraska"), ("NV", "Nevada"), + ("NH", "New Hampshire"), ("NJ", "New Jersey"), ("NM", "New Mexico"), ("NY", "New York"), + ("NC", "North Carolina"), ("ND", "North Dakota"), ("OH", "Ohio"), ("OK", "Oklahoma"), + ("OR", "Oregon"), ("PA", "Pennsylvania"), ("RI", "Rhode Island"), ("SC", "South Carolina"), + ("SD", "South Dakota"), ("TN", "Tennessee"), ("TX", "Texas"), ("UT", "Utah"), + ("VT", "Vermont"), ("VA", "Virginia"), ("WA", "Washington"), ("WV", "West Virginia"), + ("WI", "Wisconsin"), ("WY", "Wyoming"), +]; + +#[cfg(test)] +mod tests { + use super::*; + + fn p(place: &str) -> PlacePath { + normalize(Some(place), None) + } + + #[test] + fn irish_townland_resolves_the_whole_ladder() { + assert_eq!( + p("Raheen, Clashmore, Co. Waterford, Ireland"), + PlacePath { + country: Some("Ireland".into()), + admin: Some("Co. Waterford".into()), + locality: Some("Raheen".into()), + } + ); + } + + /// The fold that matters most: the corpus writes one county three ways, and a chart that + /// keeps them apart splits a branch's origin across three slices. + #[test] + fn county_synonyms_fold_together() { + for s in [ + "Cork, Co. Cork, Ireland", + "Cork, County Cork, Ireland", + "Cork, Cork, Ireland", + ] { + assert_eq!(p(s).admin.as_deref(), Some("Co. Cork"), "{s}"); + } + // The city survives as the locality — folding the admin must not consume it. + assert_eq!(p("Cork, Co. Cork, Ireland").locality.as_deref(), Some("Cork")); + } + + #[test] + fn us_states_fold_abbreviation_to_name() { + assert_eq!( + p("Pickens County, SC, USA"), + PlacePath { + country: Some("United States".into()), + admin: Some("South Carolina".into()), + locality: Some("Pickens County".into()), + } + ); + assert_eq!(p("Amelia County, Virginia, USA").admin.as_deref(), Some("Virginia")); + // Two components: the state is the admin, and there is no finer place to name. + assert_eq!( + p("Kentucky, USA"), + PlacePath { + country: Some("United States".into()), + admin: Some("Kentucky".into()), + locality: None, + } + ); + } + + #[test] + fn uk_postcodes_are_stripped_not_treated_as_places() { + assert_eq!( + p("Moulin, Pitlochry PH16 5EP, UK"), + PlacePath { + country: Some(UNITED_KINGDOM.into()), + admin: Some("Pitlochry".into()), + locality: Some("Moulin".into()), + } + ); + } + + /// A bare `UK` behind a constituent country is a geocoder artefact. Scotland vs England is + /// the distinction a Y project reads by, so the constituent country wins. + #[test] + fn constituent_country_outranks_a_trailing_uk() { + assert_eq!(p("Chelmsford, England, UK").country.as_deref(), Some("England")); + assert_eq!(p("Isle of Lewis, Scotland").country.as_deref(), Some("Scotland")); + // Nothing to promote to: no gazetteer says which country this council area sits in, and + // guessing would be worse than leaving it. + assert_eq!(p("Na h-Eileanan an Iar, UK").country.as_deref(), Some(UNITED_KINGDOM)); + } + + #[test] + fn country_only_strings_and_spelling_variants() { + assert_eq!( + p("Ireland"), + PlacePath { country: Some("Ireland".into()), admin: None, locality: None } + ); + for s in ["Republic of Ireland", "ireland", " IRELAND "] { + assert_eq!(p(s).country.as_deref(), Some("Ireland"), "{s}"); + } + } + + /// The recorded country field carries the answer when the place string has no country — and + /// the place parts below it stay usable. + #[test] + fn falls_back_to_the_recorded_country_field() { + let path = normalize(Some("Ballyvaughan, Co. Clare"), Some("Ireland")); + assert_eq!(path.country.as_deref(), Some("Ireland")); + assert_eq!(path.admin.as_deref(), Some("Co. Clare")); + assert_eq!(path.locality.as_deref(), Some("Ballyvaughan")); + + // Country alone — what the §2.3 precision ladder yields with no ancestor birth year. + assert_eq!( + normalize(None, Some("Scotland")), + PlacePath { country: Some("Scotland".into()), admin: None, locality: None } + ); + assert!(normalize(None, None).is_empty()); + } + + /// An unrecognized division is kept, not dropped: it is still a real distinction, and + /// discarding it would silently merge two origins. + #[test] + fn unknown_admin_passes_through_title_cased() { + assert_eq!(p("Bergen, hordaland, Norway").admin.as_deref(), Some("Hordaland")); + // Already mixed-case stays exactly as recorded. + assert_eq!(p("Foo, Na h-Eileanan an Iar, Scotland").admin.as_deref(), Some("Na h-Eileanan an Iar")); + } + + #[test] + fn label_at_falls_back_coarser_never_to_unknown() { + let only_country = normalize(None, Some("Ireland")); + assert_eq!(only_country.label_at(Level::Country), Some("Ireland")); + assert_eq!(only_country.label_at(Level::Admin), Some("Ireland")); + assert_eq!(only_country.label_at(Level::Locality), Some("Ireland")); + + let full = p("Raheen, Clashmore, Co. Waterford, Ireland"); + assert_eq!(full.label_at(Level::Country), Some("Ireland")); + assert_eq!(full.label_at(Level::Admin), Some("Co. Waterford")); + assert_eq!(full.label_at(Level::Locality), Some("Raheen")); + + // Nothing recorded stays nothing — the view must draw this as its own slice. + assert_eq!(normalize(None, None).label_at(Level::Country), None); + } + + /// A postcode-shaped token is narrow on purpose: ordinary words and ordinals must survive. + #[test] + fn postcode_detection_does_not_eat_real_words() { + assert!(is_uk_postcode_token("PH16")); + assert!(is_uk_postcode_token("5EP")); + assert!(!is_uk_postcode_token("Cork")); + assert!(!is_uk_postcode_token("de")); + assert!(!is_uk_postcode_token("1234")); + assert_eq!(p("Sligo, Co. Sligo, Ireland").locality.as_deref(), Some("Sligo")); + } + + /// US ZIPs left in place made every town its own admin — dozens of `Va 24521` singletons + /// across the reference corpus, all of them Virginia. + #[test] + fn us_zips_are_stripped_so_the_state_folds() { + assert!(is_us_zip_token("24521")); + assert!(is_us_zip_token("22554-7232")); + assert!(!is_us_zip_token("1783"), "a four-digit year is not a ZIP"); + assert!(!is_us_zip_token("Cork")); + + for s in ["Amherst, VA 24521, USA", "Amherst, Virginia, USA", "Amherst, VA, USA"] { + assert_eq!(p(s).admin.as_deref(), Some("Virginia"), "{s}"); + } + } + + /// A bare trailing state token is a US address. This runs after country matching, so the + /// codes that collide with countries are already claimed and cannot be stolen back. + #[test] + fn a_trailing_state_implies_the_united_states() { + let path = p("Blount Co., AL"); + assert_eq!(path.country.as_deref(), Some("United States")); + assert_eq!(path.admin.as_deref(), Some("Alabama")); + assert_eq!(path.locality.as_deref(), Some("Blount Co.")); + + // Country codes win: these must not be re-read as Canada/Germany/India state tokens. + assert_eq!(p("Toronto, CA").country.as_deref(), Some("Canada")); + assert_eq!(p("Berlin, DE").country.as_deref(), Some("Germany")); + assert_eq!(p("Mumbai, IN").country.as_deref(), Some("India")); + } + + /// A parenthetical qualifier on a country field is an ancestry note, not a country. + #[test] + fn parenthetical_country_qualifiers_are_dropped() { + assert_eq!( + normalize(None, Some("United States (Native American)")).country.as_deref(), + Some("United States") + ); + } + + /// A declared country field is trusted even when the synonym table has never heard of it. + /// Requiring recognition here dropped every origin outside the table into "no locality + /// recorded" — 21 rows of the reference corpus that plainly had one. + #[test] + fn an_unrecognized_country_field_is_taken_at_its_word() { + for c in ["Israel", "Guernsey", "Isle of Man", "Slovenia"] { + assert_eq!(normalize(None, Some(c)).country.as_deref(), Some(c), "{c}"); + } + // A place string below it still resolves against that country. + let path = normalize(Some("Havana, Cuba"), Some("Cuba")); + assert_eq!(path.country.as_deref(), Some("Cuba")); + assert_eq!(path.locality.as_deref(), Some("Havana")); + // An empty country field is still nothing. + assert!(normalize(None, Some(" ")).is_empty()); + } +} diff --git a/rust/crates/du-jobs/src/jetstream.rs b/rust/crates/du-jobs/src/jetstream.rs index 88955135..081c0bf7 100644 --- a/rust/crates/du-jobs/src/jetstream.rs +++ b/rust/crates/du-jobs/src/jetstream.rs @@ -13,7 +13,10 @@ //! from the persisted `time_us` cursor and reconnects with capped backoff; every //! upsert is idempotent + ordered, so replay overlap on reconnect is harmless. -use du_db::fed::{self, analytics, core, coverage, device_key, instrument_observation, private_variant, str_profile}; +use du_db::fed::{ + self, analytics, ancestral_origin, core, coverage, device_key, instrument_observation, private_variant, + str_profile, +}; use du_db::PgPool; use futures_util::{SinkExt, StreamExt}; use serde::Deserialize; @@ -192,6 +195,16 @@ async fn handle(pool: &PgPool, ev: &Event) -> anyhow::Result<()> { instrument_observation::upsert(pool, &build_instrument_observation(c, record)).await? } fed::NS_PRIVATE_VARIANT => private_variant::upsert(pool, &build_private_variant(c, record)).await?, + fed::NS_ANCESTRAL_ORIGIN => { + // `None` means the record failed a privacy gate (see `build_ancestral_origin`). It is + // dropped, not stored-and-hidden: a row that exists is a row a future read path can + // leak. Logged at debug so a misbehaving client is diagnosable without the rejected + // content itself reaching the log. + match build_ancestral_origin(c, record) { + Some(o) => ancestral_origin::upsert(pool, &o).await?, + None => tracing::debug!(did = %ev.did, "ancestralOrigin rejected by a privacy gate"), + } + } fed::NS_DEVICE_KEY => { // A device key with no public key is unusable — skip it rather than store a null. if let Some(public_key) = str_at(record, "publicKey") { @@ -483,6 +496,106 @@ fn build_private_variant(c: fed::Common, record: &Value) -> private_variant::Pri } } +/// The latest ancestor birth year that may carry place-level detail. A person born in 1900 is 126 +/// today; this is the check that makes "an MDKA is not living-donor PII" verifiable rather than +/// asserted. The floor rejects corrupt years rather than trusting them. +const ANCESTOR_BIRTH_YEAR_MAX: i32 = 1900; +const ANCESTOR_BIRTH_YEAR_MIN: i32 = 1000; + +/// Name particles that legitimately precede a surname. Without these, `is_surname_only` would +/// reject `van der Berg` and `de la Cruz` — real surnames — while accepting nothing extra: +/// `Thomas Michael Kane` still fails, because `Thomas` and `Michael` are not particles. +const NAME_PARTICLES: &[&str] = &[ + "van", "von", "der", "den", "de", "del", "della", "di", "da", "dos", "du", "la", "le", "les", + "mac", "mc", "st", "st.", "saint", "ter", "ten", "af", "av", "al", "bin", "ibn", "ap", "ó", + "ni", "nic", "mag", "fitz", +]; + +/// Gate 1: the record may carry a **surname**, never a given name. A client that puts +/// `"Thomas Michael Kane"` in a field labelled `surname` is leaking one, whether by bug or by +/// design, so the AppView checks rather than trusting the label. +/// +/// The rule is "one name token, optionally preceded by particles" — which accepts the surnames +/// that genuinely contain spaces and rejects a forename sequence. +fn is_surname_only(s: &str) -> bool { + let tokens: Vec<&str> = s.split_whitespace().collect(); + if tokens.is_empty() || tokens.len() > 4 { + return false; + } + tokens[..tokens.len() - 1] + .iter() + .all(|t| NAME_PARTICLES.contains(&t.to_lowercase().as_str())) +} + +/// Gate 4: coarsen to ~1 km before storage. Applied here and not only at publish, because the +/// publisher cannot be trusted to have done it — and a county-scale view cannot use finer detail +/// anyway, while a rooftop coordinate plus a surname narrows to one family. +fn coarsen(v: f64) -> f64 { + (v * 100.0).round() / 100.0 +} + +/// Build a mirrored ancestral origin, or `None` to **reject** the record. +/// +/// Rejection is deliberate and total: an unpublishable record is not stored and later hidden, +/// because a row that exists is a row some future read path can leak. See +/// `proposals/ancestral-origin-icicle.md` §2. +fn build_ancestral_origin(c: fed::Common, record: &Value) -> Option { + let biosample_ref = str_at(record, "biosampleRef"); + let ids = external_ids(record); + // No join key at all — this can never resolve to a sample, so storing it is liability with + // no benefit. + if biosample_ref.is_none() && ids.is_empty() { + return None; + } + + // Gate 1 — surname only. + let surname = str_at(record, "surname").map(|s| s.trim().to_string()).filter(|s| !s.is_empty()); + if let Some(s) = &surname { + if !is_surname_only(s) { + return None; + } + } + + // Gate 2 — the date ceiling. A year outside the plausible range is corrupt, not merely old. + let birth_year = i32_at(record, "birthYear"); + if let Some(y) = birth_year { + if !(ANCESTOR_BIRTH_YEAR_MIN..=ANCESTOR_BIRTH_YEAR_MAX).contains(&y) { + return None; + } + } + + // Gate 3 — the precision ladder. Without a birth year there is nothing establishing that this + // ancestor is long dead, so only the country may be kept: place text and coordinate are + // dropped rather than the record. + let dated = birth_year.is_some(); + let origin_place = dated.then(|| str_at(record, "originPlace")).flatten(); + // Gate 4 — coarsen whatever coordinate survives. + let (lat, lon) = match dated { + true => ( + f64_at(record, "lat").map(coarsen), + f64_at(record, "lon").map(coarsen), + ), + false => (None, None), + }; + + Some(ancestral_origin::AncestralOrigin { + biosample_ref, + external_ids: json!(ids + .iter() + .map(|(ns, v)| json!({ "namespace": ns, "value": v })) + .collect::>()), + lineage: str_at(record, "lineage"), + surname, + origin_place, + origin_country: str_at(record, "originCountry"), + birth_year, + death_year: i32_at(record, "deathYear"), + lat, + lon, + common: c, + }) +} + fn build_reconciliation(c: fed::Common, record: &Value) -> analytics::Reconciliation { let status = record.get("status").cloned().unwrap_or_else(|| json!({})); analytics::Reconciliation { @@ -775,6 +888,104 @@ mod tests { assert_eq!(p.variants.as_array().map(|a| a.len()), Some(2)); } + fn origin(extra: Value) -> Value { + let mut base = json!({ + "externalIds": [{ "namespace": "ftdna", "value": "b5163" }], + "lineage": "Y_DNA", + "surname": "Kane", + "originPlace": "Creegh South, Co. Clare, Ireland", + "originCountry": "Ireland", + "birthYear": 1830, + "deathYear": 1908, + "lat": 52.7534567, + "lon": -9.4312345 + }); + let (Value::Object(b), Value::Object(e)) = (&mut base, extra) else { unreachable!() }; + for (k, v) in e { + if v.is_null() { + b.remove(&k); + } else { + b.insert(k, v); + } + } + base + } + + #[test] + fn ancestral_origin_extracts_and_normalizes_identifiers() { + let o = build_ancestral_origin(mk_common(), &origin(json!({}))).expect("accepted"); + assert_eq!(o.surname.as_deref(), Some("Kane")); + assert_eq!(o.origin_country.as_deref(), Some("Ireland")); + assert_eq!(o.birth_year, Some(1830)); + assert_eq!(o.death_year, Some(1908)); + // Identifiers are upper/trimmed for the `core.biosample_identifier` join. + assert_eq!( + o.external_ids, + json!([{ "namespace": "FTDNA", "value": "B5163" }]) + ); + } + + /// Gate 4 — a rooftop coordinate plus a surname narrows to one family, and a county-scale + /// view cannot use the precision anyway. Coarsening at ingest, not only at publish, is what + /// makes it hold for a client that skipped it. + #[test] + fn ancestral_origin_coarsens_coordinates_to_about_a_kilometre() { + let o = build_ancestral_origin(mk_common(), &origin(json!({}))).expect("accepted"); + assert_eq!(o.lat, Some(52.75)); + assert_eq!(o.lon, Some(-9.43)); + } + + /// Gate 1 — the field is labelled `surname`, so the AppView checks that it is one. Surnames + /// that genuinely contain spaces must survive; a forename sequence must not. + #[test] + fn ancestral_origin_rejects_a_given_name_in_the_surname_field() { + for leaked in ["Thomas Michael Kane", "Thomas Kane", "John Q Public"] { + assert!( + build_ancestral_origin(mk_common(), &origin(json!({ "surname": leaked }))).is_none(), + "{leaked} must be rejected" + ); + } + for real in ["Kane", "O'Brien", "van der Berg", "de la Cruz", "Mac Donald", "Ó Súilleabháin"] { + assert!( + build_ancestral_origin(mk_common(), &origin(json!({ "surname": real }))).is_some(), + "{real} must be accepted" + ); + } + } + + /// Gate 2 — the check that makes "not living-donor PII" verifiable rather than asserted. + #[test] + fn ancestral_origin_rejects_an_ancestor_born_after_the_ceiling() { + assert!(build_ancestral_origin(mk_common(), &origin(json!({ "birthYear": 1975 }))).is_none()); + assert!(build_ancestral_origin(mk_common(), &origin(json!({ "birthYear": 1901 }))).is_none()); + assert!(build_ancestral_origin(mk_common(), &origin(json!({ "birthYear": 1900 }))).is_some()); + // Corrupt years are rejected, not treated as very old. + assert!(build_ancestral_origin(mk_common(), &origin(json!({ "birthYear": 0 }))).is_none()); + } + + /// Gate 3 — with no birth year nothing establishes that this ancestor is long dead, so the + /// country survives and the precise place does not. The record is kept; the detail is not. + #[test] + fn ancestral_origin_without_a_birth_year_keeps_country_only() { + let o = build_ancestral_origin(mk_common(), &origin(json!({ "birthYear": null }))) + .expect("kept, not rejected"); + assert_eq!(o.origin_country.as_deref(), Some("Ireland"), "country survives"); + assert_eq!(o.origin_place, None, "place text dropped"); + assert_eq!(o.lat, None, "coordinate dropped"); + assert_eq!(o.lon, None); + } + + /// A record with no way to reach a sample is pure liability: it can never be rendered, and it + /// can still be read. + #[test] + fn ancestral_origin_rejects_a_record_with_no_join_key() { + let orphan = origin(json!({ "externalIds": [], "biosampleRef": null })); + assert!(build_ancestral_origin(mk_common(), &orphan).is_none()); + // Either key alone is enough. + let by_uri = origin(json!({ "externalIds": [], "biosampleRef": "at://x/bs/1" })); + assert!(build_ancestral_origin(mk_common(), &by_uri).is_some()); + } + #[test] fn reconciliation_extracts_status_consensus() { let record = json!({ diff --git a/rust/crates/du-web/assets/main.css b/rust/crates/du-web/assets/main.css index 97f02244..9d7c94b1 100644 --- a/rust/crates/du-web/assets/main.css +++ b/rust/crates/du-web/assets/main.css @@ -173,3 +173,86 @@ code { color: #495057; background-color: #f8f9fa; } slice of viewport — give it more room and let the page scroll carry the rest. */ .tree-scroll-container { max-height: 88vh; } } + +/* ── Ancestral-origin icicle (proposals/ancestral-origin-icicle.md §5) ───────── */ +/* Colour roles live here so light/dark swap in one place and the SVG is written + against slots rather than raw hex. The palette is the validated 8-slot + categorical set — fixed order, never cycled. Slot 0 is the reserved neutral + carrying both "Other" and "no locality recorded": an absence is not an + identity, so it must not wear a categorical hue. + + Validated with the data-viz palette checker in both modes. Light mode raises a + contrast warning on three slots (aqua/yellow/magenta below 3:1 on a light + surface), which obliges relief — hence the always-on band labels and the + table view in origins.html. */ +.origins-icicle { + --surface-1: #ffffff; + --o-0: #adb5bd; /* reserved neutral: Other / no locality */ + --o-1: #2a78d6; --o-2: #eb6834; --o-3: #1baf7a; --o-4: #eda100; + --o-5: #e87ba4; --o-6: #008300; --o-7: #4a3aa7; --o-8: #e34948; + display: block; + background: var(--surface-1); +} +@media (prefers-color-scheme: dark) { + .origins-icicle { + --surface-1: #1a1a19; + --o-0: #6c757d; + --o-1: #3987e5; --o-2: #d95926; --o-3: #199e70; --o-4: #c98500; + --o-5: #d55181; --o-6: #008300; --o-7: #9085e9; --o-8: #e66767; + } +} +[data-bs-theme="dark"] .origins-icicle { + --surface-1: #1a1a19; + --o-0: #6c757d; + --o-1: #3987e5; --o-2: #d95926; --o-3: #199e70; --o-4: #c98500; + --o-5: #d55181; --o-6: #008300; --o-7: #9085e9; --o-8: #e66767; +} + +.origins-scroll { overflow-x: auto; max-height: 80vh; } + +/* The band behind the composition — visible where a branch has no men at all. */ +.origins-band-bg { fill: var(--bs-tertiary-bg, #f8f9fa); stroke: var(--bs-border-color, #dee2e6); stroke-width: 1; } +.origins-band a { cursor: pointer; } +.origins-band:hover .origins-band-bg { stroke: var(--bs-primary, #0d6efd); stroke-width: 1.5; } +/* Text wears text tokens, never the series colour. */ +.origins-band-label { fill: var(--bs-body-color, #212529); font-weight: 600; paint-order: stroke; stroke: var(--surface-1); stroke-width: 2.5px; } +.origins-tip-label { fill: var(--bs-secondary-color, #6c757d); paint-order: stroke; stroke: var(--surface-1); stroke-width: 2.5px; } + +/* Stacked segments are separated by surface, not by a stroke (mark spec). */ +.origins-seg { stroke: none; } +.origins-seg.s0 { fill: var(--o-0); } +.origins-seg.s1 { fill: var(--o-1); } +.origins-seg.s2 { fill: var(--o-2); } +.origins-seg.s3 { fill: var(--o-3); } +.origins-seg.s4 { fill: var(--o-4); } +.origins-seg.s5 { fill: var(--o-5); } +.origins-seg.s6 { fill: var(--o-6); } +.origins-seg.s7 { fill: var(--o-7); } +.origins-seg.s8 { fill: var(--o-8); } + +/* An unmeasured branch is hatched, so it can never read as a measured short one. */ +.origins-undated-bg { fill: transparent; } +.origins-undated-line { stroke: var(--bs-secondary-color, #6c757d); stroke-width: 1.5; opacity: .5; } + +/* Recessive ruler. */ +.origins-tick line { stroke: var(--bs-border-color, #dee2e6); stroke-width: 1; } +.origins-tick text { fill: var(--bs-secondary-color, #6c757d); } + +.origins-legend { display: flex; flex-wrap: wrap; gap: .25rem .9rem; } +.origins-legend-item { display: inline-flex; align-items: center; gap: .3rem; } +.origins-swatch { + display: inline-block; width: .8rem; height: .8rem; + border-radius: 2px; border: 1px solid rgba(0,0,0,.15); + --o-0: #adb5bd; + --o-1: #2a78d6; --o-2: #eb6834; --o-3: #1baf7a; --o-4: #eda100; + --o-5: #e87ba4; --o-6: #008300; --o-7: #4a3aa7; --o-8: #e34948; +} +.origins-swatch.s0 { background: var(--o-0); } +.origins-swatch.s1 { background: var(--o-1); } +.origins-swatch.s2 { background: var(--o-2); } +.origins-swatch.s3 { background: var(--o-3); } +.origins-swatch.s4 { background: var(--o-4); } +.origins-swatch.s5 { background: var(--o-5); } +.origins-swatch.s6 { background: var(--o-6); } +.origins-swatch.s7 { background: var(--o-7); } +.origins-swatch.s8 { background: var(--o-8); } diff --git a/rust/crates/du-web/src/main.rs b/rust/crates/du-web/src/main.rs index b2b32775..33f87e02 100644 --- a/rust/crates/du-web/src/main.rs +++ b/rust/crates/du-web/src/main.rs @@ -17,6 +17,7 @@ mod htmx; mod i18n; mod render; mod oauth; +mod origins_layout; mod routes; mod security; mod sig; diff --git a/rust/crates/du-web/src/origins_layout.rs b/rust/crates/du-web/src/origins_layout.rs new file mode 100644 index 00000000..72f79556 --- /dev/null +++ b/rust/crates/du-web/src/origins_layout.rs @@ -0,0 +1,844 @@ +//! Server-side layout for the **ancestral-origin icicle** — the genealogical-era companion to +//! `tree_layout`'s cladogram. Design: `proposals/ancestral-origin-icicle.md` §5. +//! +//! The shape is ytree.net's / the Big Tree's: depth runs **down** the page, each branch is a band +//! spanning the horizontal extent of its descendants, and children sit flush beneath their parent +//! so *containment* carries descent and no connector is drawn. What differs is the fill — instead +//! of the branch's SNPs, a band is a **stacked composition of where its men's ancestors came +//! from**. +//! +//! **Time is absolute, not cumulative.** A band's top and bottom are dates on one linear axis, so +//! its height *is* its duration; nothing accumulates and nothing drifts. +//! +//! A branch spans **its parent's TMRCA → its own TMRCA**: from the split that brought it into +//! existence as a separate line, to the point it began diversifying itself. That specific pairing +//! is deliberate. The obvious choice — a node's own `formed_ybp` → its own `tmrca_ybp` — draws +//! children *above* their parents on real data: `formed_ybp` and the parent's `tmrca_ybp` are +//! independent point estimates under no monotonicity constraint, and on the live Y tree they are +//! equal on only 898 of 10,252 edges while **4,243 (41%) have the child forming earlier than its +//! parent's split**. Parent-TMRCA → own-TMRCA has zero inversions across the same 10,252 edges, so +//! containment is guaranteed by construction rather than by hope. `formed_ybp` is still reported, +//! on the band itself, where a reader can see the estimate without the geometry depending on it. +//! +//! Undated nodes (16% of sample-bearing nodes) are **not** silently normalized: they hang from +//! their parent at a minimum height and are hatched, so an unmeasured branch never reads as a +//! short one. +//! +//! Pure: no DB, no `Ui`, no template. Every function here is testable without a canvas. + +use du_db::origins::SampleOrigin; +use du_db::place::Level; +use std::collections::HashMap; + +/// Canvas geometry at scale 1. +const LEAF_W: f64 = 74.0; +const H_GAP: f64 = 4.0; +/// Pixels per year of elapsed time. The genealogical era is ~1,500 years, so this puts a full +/// gated subtree in roughly 900px. +const PX_PER_YEAR: f64 = 0.6; +/// A band never collapses below this, however brief the branch — a 20-year branch must still be +/// clickable and still show its composition. +const MIN_BAND_H: f64 = 18.0; +/// Height given to a band with no age at all. Deliberately equal to the minimum so it cannot be +/// mistaken for a *measured* short branch; the hatch is what distinguishes it. +const UNDATED_H: f64 = MIN_BAND_H; +/// Sample tips hang in a band below the youngest branch. +const TIP_H: f64 = 16.0; +const TIP_GAP: f64 = 10.0; +const GUTTER_W: f64 = 54.0; +const MARGIN: f64 = 8.0; +/// Gap between stacked segments, per the mark spec — segments are separated by surface, not by a +/// stroke. +const SEG_GAP: f64 = 2.0; + +/// Categorical slots available before folding into "Other". The palette is fixed-order and never +/// cycled; a ninth locality is not given a generated hue. +pub const MAX_SERIES: usize = 8; + +/// One locality's share of a band. `slot` indexes the fixed categorical palette (1..=[`MAX_SERIES`]); +/// `0` is the reserved neutral used for both "Other" and "no locality recorded", which are +/// absences rather than identities and must not wear a categorical hue. +#[derive(Debug, Clone, PartialEq)] +pub struct Segment { + pub label: Option, + pub count: usize, + pub slot: usize, + pub x: f64, + pub w: f64, +} + +/// One laid-out branch. +#[derive(Debug, Clone, PartialEq)] +pub struct Band { + pub id: i64, + pub name: String, + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, + /// False when the branch has no age estimate — rendered hatched, excluded from the ruler. + pub dated: bool, + pub formed_ybp: Option, + pub tmrca_ybp: Option, + /// Placed samples at or below this branch that carry a published origin. + pub with_origin: usize, + /// Placed samples at or below it that do not — always drawn, never omitted. + pub without_origin: usize, + pub segments: Vec, + /// True when the band is too short to letter — the view puts its label in the tooltip only. + pub cramped: bool, +} + +/// One man, as a leaf below the branch he is placed on. +#[derive(Debug, Clone, PartialEq)] +pub struct Tip { + pub label: String, + pub slot: usize, + pub x: f64, + pub y: f64, + pub w: f64, + pub h: f64, +} + +/// A ruler graduation on the absolute time axis. +#[derive(Debug, Clone, PartialEq)] +pub struct Tick { + pub y: f64, + pub ybp: i32, + /// Calendar-era label (`"1500 CE"`), because the genealogical era reads in calendar years. + pub label: String, +} + +/// A legend row. Present whenever the chart carries two or more series — identity is never +/// carried by colour alone. +#[derive(Debug, Clone, PartialEq)] +pub struct LegendEntry { + pub label: Option, + pub slot: usize, + pub count: usize, +} + +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Laid { + pub width: f64, + pub height: f64, + pub bands: Vec, + pub tips: Vec, + pub ticks: Vec, + pub legend: Vec, + /// Samples under the root with no published origin — the honest denominator. + pub unresolved: usize, + /// Branches dropped because no origin sits beneath them. Reported, never silent: a pruned + /// chart that looked complete would misrepresent how much of the clade this is. + pub pruned: usize, +} + +/// The minimum a node needs from the tree window. Mirrors `du_db::haplogroup::WindowNode` so this +/// module stays independent of the DB layer and testable with literals. +#[derive(Debug, Clone, PartialEq)] +pub struct Node { + pub id: i64, + pub name: String, + pub parent_id: Option, + pub formed_ybp: Option, + pub tmrca_ybp: Option, + /// A de-novo auto-named node, which must not surface publicly. It stays in the input so the + /// ancestor walk is unbroken, and its men are attributed to the nearest named ancestor. + pub hidden: bool, +} + +/// Attribute every origin to the nearest **visible** branch at or above where its sample is +/// placed, and drop the branches that carry no origin at all. +/// +/// Both halves fix silent losses found by rendering the real tree: +/// +/// - A sample placed on a hidden (de-novo) node, or below the drawn window, used to contribute to +/// **nothing** — its ancestors never saw it, so every band above it understated its own +/// composition. Climbing to the nearest visible ancestor is what the tree's own private-node +/// collapse does for sample tips, applied to composition. +/// - An origins view is about origins: a branch with none beneath it costs a full column of width +/// and says nothing. On a real clade that was 175 bands and a 7,944px canvas for 10 origins. +/// Pruning is reported, never silent. +/// +/// Returns the retained nodes (root always kept), the origins re-pointed at visible branches, and +/// how many branches were pruned. +pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> (Vec, Vec, usize) { + let by_id: HashMap = nodes.iter().map(|n| (n.id, n)).collect(); + let Some(root) = nodes.iter().find(|n| n.parent_id.is_none()) else { + return (Vec::new(), Vec::new(), 0); + }; + + // Climb to the nearest visible ancestor. The root is the floor: it is always drawn, so no + // origin can escape the chart entirely. + let visible_ancestor = |mut at: i64| -> i64 { + let mut guard = 0; + while let Some(n) = by_id.get(&at) { + if !n.hidden { + return at; + } + match n.parent_id { + Some(p) if guard < nodes.len() => { + at = p; + guard += 1; + } + _ => break, + } + } + root.id + }; + let moved: Vec = origins + .iter() + .map(|o| SampleOrigin { + haplogroup_id: match by_id.contains_key(&o.haplogroup_id) { + true => visible_ancestor(o.haplogroup_id), + // Placed below the drawn window: attribute to the root rather than lose it. + false => root.id, + }, + ..o.clone() + }) + .collect(); + + // Keep a branch when an origin sits at or below it. + let mut keep: std::collections::HashSet = std::collections::HashSet::new(); + keep.insert(root.id); + for o in &moved { + let mut at = Some(o.haplogroup_id); + let mut guard = 0; + while let Some(id) = at { + if guard > nodes.len() { + break; + } + keep.insert(id); + at = by_id.get(&id).and_then(|n| n.parent_id); + guard += 1; + } + } + // Re-parent onto the nearest retained ancestor. Dropping a hidden or origin-less branch must + // not orphan the branches beneath it — the chain has to stay walkable or the roll-up and the + // layout both lose everything below the gap. + let retained_id = |id: i64| -> bool { !by_id[&id].hidden && keep.contains(&id) }; + let retained: Vec = nodes + .iter() + .filter(|n| !n.hidden && keep.contains(&n.id)) + .map(|n| { + let mut p = n.parent_id; + let mut guard = 0; + while let Some(pid) = p { + if !by_id.contains_key(&pid) || guard > nodes.len() { + p = None; + break; + } + if retained_id(pid) { + break; + } + p = by_id[&pid].parent_id; + guard += 1; + } + Node { parent_id: p, ..n.clone() } + }) + .collect(); + let pruned = nodes.iter().filter(|n| !n.hidden).count() - retained.len(); + (retained, moved, pruned) +} + +/// Roll each sample's locality up to its branch **and every ancestor of that branch**, so a band's +/// composition is what lies beneath it rather than what sits exactly on it. +/// +/// Returns `node id -> (label -> count)`, where `None` is "no locality recorded" — kept as a key +/// rather than dropped, because a branch whose men are mostly unrecorded and a branch whose men +/// are mostly Irish must not look alike. +pub fn roll_up( + nodes: &[Node], + origins: &[SampleOrigin], + level: Level, +) -> HashMap, usize>> { + let parent: HashMap> = nodes.iter().map(|n| (n.id, n.parent_id)).collect(); + let mut out: HashMap, usize>> = HashMap::new(); + for o in origins { + let label = o.place.label_at(level).map(str::to_string); + // Climb to the root. A sample outside this window contributes nothing, which is correct: + // the window is the subtree being drawn. + let mut at = Some(o.haplogroup_id); + let mut guard = 0; + while let Some(id) = at { + if !parent.contains_key(&id) || guard > nodes.len() { + break; + } + *out.entry(id).or_default().entry(label.clone()).or_default() += 1; + at = parent[&id]; + guard += 1; + } + } + out +} + +/// Assign a palette slot to each locality, **ranked once over the whole subtree** and then held +/// fixed for every band on the page. +/// +/// Ranking per band would repaint a locality as you moved down the tree, and ranking per rendered +/// view would repaint the survivors when the reader re-roots — both violate the rule that colour +/// follows the entity, not its position. Ties break on the label so the assignment is +/// deterministic across requests. +/// +/// Slot `0` is reserved: it takes "no locality recorded" and everything past [`MAX_SERIES`], which +/// fold together visually as *absence of a named origin* rather than being given invented hues. +pub fn assign_slots(root_composition: &HashMap, usize>) -> HashMap { + let mut named: Vec<(&String, &usize)> = root_composition + .iter() + .filter_map(|(k, v)| k.as_ref().map(|k| (k, v))) + .collect(); + named.sort_by(|a, b| b.1.cmp(a.1).then_with(|| a.0.cmp(b.0))); + named + .into_iter() + .take(MAX_SERIES) + .enumerate() + .map(|(i, (label, _))| (label.clone(), i + 1)) + .collect() +} + +/// Order a band's composition into drawable segments: named localities by descending count (ties +/// on label), then "other", then "no locality recorded" last so absence always sits at the same +/// end of every bar and the eye can compare bands. +fn segments_for( + comp: &HashMap, usize>, + slots: &HashMap, +) -> (Vec, usize, usize) { + let mut named: Vec<(&String, usize)> = Vec::new(); + let mut other = 0usize; + let mut unknown = 0usize; + for (label, n) in comp { + match label { + None => unknown += *n, + Some(l) => match slots.get(l) { + Some(_) => named.push((l, *n)), + None => other += *n, + }, + } + } + named.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0))); + + let mut segs: Vec = named + .into_iter() + .map(|(l, n)| Segment { + slot: slots[l], + label: Some(l.clone()), + count: n, + x: 0.0, + w: 0.0, + }) + .collect(); + let with_origin: usize = segs.iter().map(|s| s.count).sum::() + other; + if other > 0 { + segs.push(Segment { label: None, count: other, slot: 0, x: 0.0, w: 0.0 }); + } + (segs, with_origin, unknown) +} + +/// Lay the subtree out. `nodes` is the tree window (parents before children is not required); +/// `origins` are the published origins of the placed samples beneath it. +pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, placed_total: usize) -> Laid { + // Attribute origins to visible branches and drop the branches with none beneath them, before + // anything is measured — see `prune_to_origins`. + let (nodes, origins, pruned) = prune_to_origins(all_nodes, all_origins); + let (nodes, origins) = (&nodes[..], &origins[..]); + let Some(root) = nodes.iter().find(|n| n.parent_id.is_none()) else { + return Laid::default(); + }; + let comp = roll_up(nodes, origins, level); + let root_comp = comp.get(&root.id).cloned().unwrap_or_default(); + let slots = assign_slots(&root_comp); + + let index: HashMap = nodes.iter().enumerate().map(|(i, n)| (n.id, i)).collect(); + let mut children: Vec> = vec![Vec::new(); nodes.len()]; + for (i, n) in nodes.iter().enumerate() { + if let Some(p) = n.parent_id.and_then(|p| index.get(&p)) { + children[*p].push(i); + } + } + // Stable draw order: by name, so the same tree lays out the same way on every request. + for kids in &mut children { + kids.sort_by(|&a, &b| nodes[a].name.cmp(&nodes[b].name)); + } + let root_i = index[&root.id]; + + // Pass 1 (post-order): horizontal extent each subtree needs. + let mut extent = vec![LEAF_W; nodes.len()]; + let order = post_order(&children, root_i); + for &i in &order { + if children[i].is_empty() { + continue; + } + let kids: f64 = children[i].iter().map(|&c| extent[c]).sum(); + let gaps = H_GAP * (children[i].len() - 1) as f64; + extent[i] = extent[i].max(kids + gaps); + } + + // The time axis. The root's formation is the top of the canvas; the present is the bottom, so + // tips land on "now" and every band sits at its true date. + let top_ybp = root.formed_ybp.or(root.tmrca_ybp).unwrap_or(0); + let y_of = |ybp: i32| MARGIN + (top_ybp - ybp).max(0) as f64 * PX_PER_YEAR; + // A branch begins at its parent's TMRCA — the split that created it. See the module header for + // why this is not the node's own `formed_ybp`. + let tmrca_of: HashMap = nodes.iter().filter_map(|n| Some((n.id, n.tmrca_ybp?))).collect(); + let split_of: HashMap = nodes + .iter() + .filter_map(|n| Some((n.id, *tmrca_of.get(&n.parent_id?)?))) + .collect(); + + // Pass 2 (pre-order): x from the parent's band, y from the node's own dates. + let mut bands = Vec::with_capacity(nodes.len()); + let mut tips = Vec::new(); + let mut left = vec![0.0f64; nodes.len()]; + let mut fallback_top = vec![0.0f64; nodes.len()]; + left[root_i] = GUTTER_W; + fallback_top[root_i] = MARGIN; + let mut stack = vec![root_i]; + let mut deepest = 0.0f64; + while let Some(i) = stack.pop() { + let n = &nodes[i]; + // Top: the parent's TMRCA for a child, the node's own formation for the root. Bottom: this + // node's TMRCA. The pairing is monotone, so `h` can never come out negative. + let top_ybp_of = split_of.get(&n.id).copied().or(n.formed_ybp); + let dated = n.tmrca_ybp.is_some(); + let (y, h) = match (top_ybp_of, n.tmrca_ybp) { + (Some(from), Some(to)) => (y_of(from), (y_of(to) - y_of(from)).max(MIN_BAND_H)), + // Undated: hang from wherever the parent ended, at the minimum height. Hatched, so it + // reads as unmeasured rather than brief. + _ => (fallback_top[i], UNDATED_H), + }; + let (mut segs, with_origin, without_origin) = + segments_for(comp.get(&n.id).unwrap_or(&HashMap::new()), &slots); + + // Widths proportional to the composition, with a surface gap between segments. + let inner = extent[i]; + let total = with_origin + without_origin; + if total > 0 { + let gaps = SEG_GAP * segs.len().saturating_sub(1) as f64; + let usable = (inner - gaps).max(0.0); + let mut x = left[i]; + for s in &mut segs { + s.w = usable * (s.count as f64 / total as f64); + s.x = x; + x += s.w + SEG_GAP; + } + } + bands.push(Band { + id: n.id, + name: n.name.clone(), + x: left[i], + y, + w: extent[i], + h, + dated, + formed_ybp: n.formed_ybp, + tmrca_ybp: n.tmrca_ybp, + with_origin, + without_origin, + segments: segs, + cramped: h < 14.0, + }); + deepest = deepest.max(y + h); + + let mut cx = left[i]; + for &c in &children[i] { + left[c] = cx; + fallback_top[c] = y + h; + cx += extent[c] + H_GAP; + stack.push(c); + } + } + + // Tips: one per sample with a published origin, under the branch it sits on. + let tip_y = deepest + TIP_GAP; + let mut per_node: HashMap> = HashMap::new(); + for o in origins { + per_node.entry(o.haplogroup_id).or_default().push(o); + } + let band_x: HashMap = bands.iter().map(|b| (b.id, (b.x, b.w))).collect(); + for (node_id, mut list) in per_node { + let Some(&(bx, bw)) = band_x.get(&node_id) else { continue }; + list.sort_by(|a, b| a.sample_guid.cmp(&b.sample_guid)); + let n = list.len() as f64; + let w = ((bw - H_GAP * (n - 1.0).max(0.0)) / n).min(LEAF_W).max(8.0); + for (k, o) in list.iter().enumerate() { + let locality = o.place.label_at(level); + let label = match (&o.surname, locality) { + (Some(s), Some(l)) => format!("{s} · {l}"), + (Some(s), None) => s.clone(), + (None, Some(l)) => l.to_string(), + (None, None) => String::new(), + }; + tips.push(Tip { + slot: locality.and_then(|l| slots.get(l).copied()).unwrap_or(0), + label, + x: bx + k as f64 * (w + H_GAP), + y: tip_y, + w, + h: TIP_H, + }); + } + } + + let ticks = ruler(top_ybp, deepest, &y_of); + let legend = legend_for(&root_comp, &slots); + let width = GUTTER_W + extent[root_i] + MARGIN * 2.0; + let height = tip_y + TIP_H + MARGIN; + let resolved: usize = root_comp.values().sum(); + Laid { + width, + height, + bands, + tips, + ticks, + legend, + unresolved: placed_total.saturating_sub(resolved), + pruned, + } +} + +/// Children before parents, iteratively — the tree is user-shaped and may be deep enough to blow +/// a recursive stack. +fn post_order(children: &[Vec], root: usize) -> Vec { + let mut out = Vec::with_capacity(children.len()); + let mut stack = vec![root]; + while let Some(i) = stack.pop() { + out.push(i); + stack.extend(children[i].iter().copied()); + } + out.reverse(); + out +} + +/// Graduations at a round interval chosen so the axis carries roughly 6–10 of them. +fn ruler(top_ybp: i32, bottom_px: f64, y_of: &dyn Fn(i32) -> f64) -> Vec { + if top_ybp <= 0 { + return Vec::new(); + } + let step = [50, 100, 200, 250, 500, 1000, 2000, 5000] + .into_iter() + .find(|s| top_ybp / s <= 10) + .unwrap_or(10_000); + let mut ticks = Vec::new(); + let mut ybp = (top_ybp / step) * step; + while ybp >= 0 { + let y = y_of(ybp); + if y <= bottom_px + 1.0 { + ticks.push(Tick { y, ybp, label: era_label(ybp) }); + } + ybp -= step; + } + ticks +} + +/// `ybp` → a calendar-era label. The genealogical era is read in calendar years, not in years +/// before present, and 1950 is the radiocarbon reference the rest of the tree uses. +fn era_label(ybp: i32) -> String { + let year = 1950 - ybp; + if year > 0 { + format!("{year} CE") + } else { + format!("{} BCE", 1 - year) + } +} + +fn legend_for(root_comp: &HashMap, usize>, slots: &HashMap) -> Vec { + let (segs, _, unknown) = segments_for(root_comp, slots); + let mut out: Vec = segs + .into_iter() + .map(|s| LegendEntry { label: s.label, slot: s.slot, count: s.count }) + .collect(); + if unknown > 0 { + out.push(LegendEntry { label: None, slot: 0, count: unknown }); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use du_db::place::{self, PlacePath}; + use uuid::Uuid; + + fn node(id: i64, name: &str, parent: Option, formed: Option, tmrca: Option) -> Node { + Node { id, name: name.into(), parent_id: parent, formed_ybp: formed, tmrca_ybp: tmrca, hidden: false } + } + + fn hidden(id: i64, name: &str, parent: Option, formed: Option, tmrca: Option) -> Node { + Node { hidden: true, ..node(id, name, parent, formed, tmrca) } + } + + fn origin(node_id: i64, place: &str) -> SampleOrigin { + SampleOrigin { + sample_guid: Uuid::new_v4(), + haplogroup_id: node_id, + surname: Some("Kane".into()), + place: place::normalize(Some(place), None), + birth_year: Some(1830), + } + } + + fn bare(node_id: i64) -> SampleOrigin { + SampleOrigin { + sample_guid: Uuid::new_v4(), + haplogroup_id: node_id, + surname: Some("Walsh".into()), + place: PlacePath::default(), + birth_year: None, + } + } + + fn tree() -> Vec { + vec![ + node(1, "R-S764", None, Some(1600), Some(1355)), + node(2, "R-A", Some(1), Some(1355), Some(800)), + node(3, "R-B", Some(1), Some(1355), Some(600)), + ] + } + + /// A band shows what lies *beneath* it, not what sits exactly on it — otherwise every interior + /// branch would read as empty. + #[test] + fn composition_rolls_up_to_every_ancestor() { + let origins = vec![origin(2, "Cork, Co. Cork, Ireland"), origin(3, "Kenmare, Co. Kerry, Ireland")]; + let comp = roll_up(&tree(), &origins, Level::Admin); + assert_eq!(comp[&1].values().sum::(), 2, "root sees both"); + assert_eq!(comp[&2][&Some("Co. Cork".into())], 1); + assert_eq!(comp[&3][&Some("Co. Kerry".into())], 1); + assert!(!comp[&2].contains_key(&Some("Co. Kerry".into())), "a sibling's origin is not borrowed"); + } + + /// "No locality recorded" is a key, not a dropped row: a branch whose men are unrecorded and a + /// branch whose men are Irish must not look alike. + #[test] + fn samples_without_a_locality_are_counted_not_dropped() { + let origins = vec![origin(2, "Cork, Co. Cork, Ireland"), bare(2)]; + let comp = roll_up(&tree(), &origins, Level::Admin); + assert_eq!(comp[&2][&None], 1); + assert_eq!(comp[&2].values().sum::(), 2); + + let laid = layout(&tree(), &origins, Level::Admin, 2); + let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); + assert_eq!(root.with_origin, 1); + assert_eq!(root.without_origin, 1, "drawn, never omitted"); + } + + /// Colour follows the entity. Ranking is done once over the root and held, so drilling into a + /// child cannot repaint a locality that survived. + #[test] + fn slots_are_stable_and_rank_by_count_then_name() { + let mut comp = HashMap::new(); + comp.insert(Some("Co. Cork".to_string()), 5); + comp.insert(Some("Co. Kerry".to_string()), 2); + comp.insert(Some("Co. Clare".to_string()), 2); // ties Kerry — name breaks it + comp.insert(None, 9); // the unknown pile never takes a categorical slot + let slots = assign_slots(&comp); + assert_eq!(slots["Co. Cork"], 1); + assert_eq!(slots["Co. Clare"], 2, "tie broken on label, deterministically"); + assert_eq!(slots["Co. Kerry"], 3); + assert_eq!(slots.len(), 3, "None is not assigned a hue"); + } + + /// A ninth locality is not given an invented hue — it folds into the reserved neutral. + #[test] + fn past_eight_localities_fold_into_the_reserved_slot() { + let comp: HashMap, usize> = + (0..12).map(|i| (Some(format!("Place {i:02}")), 12 - i)).collect(); + let slots = assign_slots(&comp); + assert_eq!(slots.len(), MAX_SERIES); + assert!(slots.values().all(|&s| (1..=MAX_SERIES).contains(&s))); + + let (segs, with_origin, _) = segments_for(&comp, &slots); + let folded = segs.iter().find(|s| s.slot == 0).expect("an Other segment exists"); + assert_eq!(folded.label, None); + assert_eq!(with_origin, comp.values().sum::()); + } + + /// Height is duration on one absolute axis: a branch that ran twice as long is twice as tall, + /// wherever it sits in the tree. + #[test] + fn band_height_is_elapsed_time_on_an_absolute_axis() { + let laid = layout(&tree(), &[origin(2, "Ireland"), origin(3, "Scotland")], Level::Country, 2); + let b = |id: i64| laid.bands.iter().find(|b| b.id == id).unwrap().clone(); + // Both children begin at the parent's split (1355) and run to their own TMRCA. + assert!((b(2).h - 555.0 * PX_PER_YEAR).abs() < 0.01, "1355→800"); + assert!((b(3).h - 755.0 * PX_PER_YEAR).abs() < 0.01, "1355→600"); + // Siblings share that split, so their tops align exactly. + assert!((b(2).y - b(3).y).abs() < 0.01); + // And each child starts where the parent's diversification did. + assert!((b(2).y - (b(1).y + b(1).h)).abs() < 0.01); + } + + /// The bug that rendering the real tree exposed. `formed_ybp` and the parent's `tmrca_ybp` are + /// independent estimates under no monotonicity constraint: on the live Y tree 4,243 of 10,252 + /// edges have a child forming *earlier* than its parent's split. Driving the geometry from + /// `formed_ybp` drew those children on top of their parents — R-A13318 landed at exactly its + /// parent R-S764's y, at the top of the canvas. + /// + /// Spanning parent-TMRCA → own-TMRCA cannot do that, whatever `formed_ybp` says. + #[test] + fn a_child_forming_before_its_parents_split_still_nests() { + // R-S764 / R-A13318's real numbers. + let nodes = vec![ + node(1, "R-S764", None, Some(1620), Some(1355)), + node(2, "R-A13318", Some(1), Some(1622), Some(1355)), // formed 2 yrs BEFORE the parent + ]; + let laid = layout(&nodes, &[origin(2, "Ireland")], Level::Country, 1); + let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); + let child = laid.bands.iter().find(|b| b.id == 2).unwrap(); + + assert!(child.y >= root.y + root.h - 0.01, "the child begins at or below the parent's split"); + assert!(child.h >= 0.0, "and never inverts into a negative height"); + assert!(child.y > root.y, "it is not drawn on top of its parent"); + } + + /// The invariant, stated once over an awkward tree: no band may start above its parent's end. + #[test] + fn no_band_ever_starts_above_its_parent() { + let nodes = vec![ + node(1, "R-Root", None, Some(2000), Some(1500)), + node(2, "R-Early", Some(1), Some(1900), Some(1200)), // formed long before the split + node(3, "R-Late", Some(1), Some(1400), Some(900)), + node(4, "R-Deep", Some(2), Some(1800), Some(400)), // ditto, one level down + ]; + let origins: Vec<_> = [2, 3, 4].iter().map(|&id| origin(id, "Ireland")).collect(); + let laid = layout(&nodes, &origins, Level::Country, 3); + let by_id: HashMap = laid.bands.iter().map(|b| (b.id, b)).collect(); + for n in &nodes { + let (Some(b), Some(p)) = (by_id.get(&n.id), n.parent_id.and_then(|p| by_id.get(&p))) else { + continue; + }; + assert!(b.y >= p.y + p.h - 0.01, "{} starts above its parent", n.name); + } + } + + /// An unmeasured branch must not read as a short one. + #[test] + fn an_undated_branch_is_hatched_at_the_minimum_height() { + let mut nodes = tree(); + nodes.push(node(4, "R-C", Some(2), None, None)); + let laid = layout(&nodes, &[origin(4, "Ireland")], Level::Country, 1); + let c = laid.bands.iter().find(|b| b.id == 4).unwrap(); + assert!(!c.dated); + assert_eq!(c.h, UNDATED_H); + // It hangs off its parent rather than floating at the top of the canvas. + let parent = laid.bands.iter().find(|b| b.id == 2).unwrap(); + assert!((c.y - (parent.y + parent.h)).abs() < 0.01); + } + + /// Containment carries descent: a parent spans its children, and siblings never overlap. + #[test] + fn a_parent_spans_its_children_and_siblings_do_not_overlap() { + let laid = layout(&tree(), &[origin(2, "Ireland"), origin(3, "Scotland")], Level::Country, 2); + let b = |id: i64| laid.bands.iter().find(|b| b.id == id).unwrap().clone(); + let (root, a, bb) = (b(1), b(2), b(3)); + assert!(a.x >= root.x && a.x + a.w <= root.x + root.w + 0.01); + assert!(bb.x >= root.x && bb.x + bb.w <= root.x + root.w + 0.01); + assert!(a.x + a.w <= bb.x + 0.01, "siblings are disjoint"); + } + + /// Segment widths are proportional and stay inside the band, gaps included. + #[test] + fn segments_are_proportional_and_stay_within_the_band() { + let origins = vec![ + origin(2, "Cork, Co. Cork, Ireland"), + origin(2, "Cork, Co. Cork, Ireland"), + origin(3, "Kenmare, Co. Kerry, Ireland"), + bare(3), + ]; + let laid = layout(&tree(), &origins, Level::Admin, 4); + let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); + let cork = root.segments.iter().find(|s| s.label.as_deref() == Some("Co. Cork")).unwrap(); + let kerry = root.segments.iter().find(|s| s.label.as_deref() == Some("Co. Kerry")).unwrap(); + assert!((cork.w / kerry.w - 2.0).abs() < 0.01, "2 Cork to 1 Kerry"); + for s in &root.segments { + assert!(s.x >= root.x - 0.01 && s.x + s.w <= root.x + root.w + 0.01); + } + } + + /// The reader is told what the chart could not account for. + #[test] + fn unresolved_samples_are_reported_against_the_placed_total() { + let laid = layout(&tree(), &[origin(2, "Ireland")], Level::Country, 17); + assert_eq!(laid.unresolved, 16, "17 placed, 1 with a published origin"); + } + + #[test] + fn the_ruler_reads_in_calendar_years() { + let laid = layout(&tree(), &[], Level::Country, 0); + assert!(!laid.ticks.is_empty()); + assert!(laid.ticks.windows(2).all(|w| w[0].y < w[1].y), "monotone down the page"); + assert_eq!(era_label(1600), "350 CE"); + assert_eq!(era_label(0), "1950 CE"); + assert_eq!(era_label(2000), "51 BCE"); + } + + /// A legend is always available for two or more series — identity is never colour alone. + #[test] + fn the_legend_covers_every_drawn_series_including_absence() { + let origins = vec![origin(2, "Cork, Co. Cork, Ireland"), bare(3)]; + let laid = layout(&tree(), &origins, Level::Admin, 2); + assert!(laid.legend.iter().any(|e| e.label.as_deref() == Some("Co. Cork"))); + assert!(laid.legend.iter().any(|e| e.label.is_none() && e.slot == 0)); + } + + #[test] + fn an_empty_window_lays_out_to_nothing_rather_than_panicking() { + assert_eq!(layout(&[], &[], Level::Country, 0), Laid::default()); + } + + /// Found by rendering the real tree: a clade drew 175 bands across 7,944px to show 10 + /// origins. A branch with none beneath it is all width and no information. + #[test] + fn branches_with_no_origin_beneath_them_are_pruned_and_counted() { + let mut nodes = tree(); + for id in 10..20 { + nodes.push(node(id, &format!("R-Empty{id}"), Some(3), Some(600), Some(400))); + } + let laid = layout(&nodes, &[origin(2, "Ireland")], Level::Country, 1); + // Root + the one branch carrying the origin. R-B and its ten empty children are gone. + assert_eq!(laid.bands.len(), 2); + assert!(laid.bands.iter().all(|b| b.id == 1 || b.id == 2)); + assert_eq!(laid.pruned, 11, "reported, never silent"); + assert!(laid.width < 200.0, "canvas follows the data, not the tree"); + } + + /// Also found by rendering: a sample on a de-novo node contributed to *nothing*, so every + /// band above it understated itself. It must be attributed to the nearest named branch. + #[test] + fn a_sample_on_a_hidden_node_is_attributed_to_the_nearest_named_branch() { + let mut nodes = tree(); + nodes.push(hidden(9, "R-(hs1)chrY:2561207 TA->T", Some(2), Some(800), Some(400))); + let laid = layout(&nodes, &[origin(9, "Cork, Co. Cork, Ireland")], Level::Admin, 1); + + assert!(laid.bands.iter().all(|b| b.id != 9), "the de-novo node never surfaces"); + let named = laid.bands.iter().find(|b| b.id == 2).expect("its named parent is drawn"); + assert_eq!(named.with_origin, 1, "its man is counted here, not lost"); + let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); + assert_eq!(root.with_origin, 1, "and still rolls up to the root"); + } + + /// A sample placed deeper than the walk still has to land somewhere, or the chart quietly + /// undercounts. + #[test] + fn a_sample_below_the_window_falls_back_to_the_root() { + let laid = layout(&tree(), &[origin(999, "Ireland")], Level::Country, 1); + let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); + assert_eq!(root.with_origin, 1); + } + + /// Dropping a branch must not orphan what hangs beneath it. + #[test] + fn pruning_reparents_rather_than_orphaning_descendants() { + let mut nodes = tree(); + nodes.push(hidden(9, "R-(hs1)chrY:99", Some(3), Some(600), Some(500))); + nodes.push(node(10, "R-Deep", Some(9), Some(500), Some(300))); + let laid = layout(&nodes, &[origin(10, "Ireland")], Level::Country, 1); + + let deep = laid.bands.iter().find(|b| b.id == 10).expect("kept: it carries an origin"); + // Its hidden parent is gone, so it must now hang off R-B, not float free. + let b3 = laid.bands.iter().find(|b| b.id == 3).unwrap(); + assert!(deep.x >= b3.x && deep.x + deep.w <= b3.x + b3.w + 0.01); + // And it sits below R-B on the time axis rather than at the top of the canvas. + assert!(deep.y >= b3.y + b3.h - 0.01); + } +} diff --git a/rust/crates/du-web/src/routes/tree.rs b/rust/crates/du-web/src/routes/tree.rs index 003002f1..478cd1e2 100644 --- a/rust/crates/du-web/src/routes/tree.rs +++ b/rust/crates/du-web/src/routes/tree.rs @@ -14,7 +14,9 @@ use crate::htmx::{HxHeaders, HxRequest}; use crate::i18n::{Locale, T}; use crate::render::html; use crate::state::AppState; +use crate::origins_layout; use crate::tree_layout::{self, InNode, Laid, Orientation, SampleTip}; +use du_db::place; use crate::auth::Curator; use axum::extract::{Path, Query, State}; use axum::http::header::{HeaderMap, HeaderValue, COOKIE, SET_COOKIE}; @@ -51,6 +53,9 @@ pub fn router() -> Router { .route("/mtree/node/:name/geo", get(mt_clade_geo)) .route("/ytree/node/:name/geo-data", get(y_clade_geo_data)) .route("/mtree/node/:name/geo-data", get(mt_clade_geo_data)) + // Genealogical-era ancestral-origin icicle (proposals/ancestral-origin-icicle.md). + .route("/ytree/node/:name/origins", get(y_origins)) + .route("/mtree/node/:name/origins", get(mt_origins)) // Curator triage for sample leaves whose published call didn't resolve to a node. .route("/manage/tree-sample/unplaced", get(unplaced)) .route("/manage/tree-sample/place", post(place)) @@ -518,6 +523,168 @@ async fn clade_geo_data( Ok(Json(json!({ "type": "FeatureCollection", "features": features }))) } +/// Ancestral origins are only meaningful in the genealogical era: past this, a band aggregates to +/// a continent and says nothing about where a *line* went. Nodes older than this render as a +/// signpost down to their eligible children rather than as a chart. +const ORIGINS_MAX_YBP: i32 = 1500; +/// Safety bound on the subtree walk — not a display choice. The drawn depth is decided by +/// `origins_layout::prune_to_origins`, which keeps only the branches carrying an origin. +const ORIGINS_DEPTH: i32 = 40; + +#[derive(Deserialize)] +struct OriginsQuery { + /// `country` (default) · `admin` · `place`. + level: Option, +} + +fn origins_level(s: Option<&str>) -> place::Level { + match s.map(str::trim).unwrap_or("") { + "admin" | "region" => place::Level::Admin, + "place" | "locality" => place::Level::Locality, + _ => place::Level::Country, + } +} + +async fn y_origins( + st: State, + locale: Locale, + user: crate::auth::MaybeUser, + name: Path, + q: Query, +) -> Result { + origins(st, locale, user, name, q, DnaType::YDna).await +} + +async fn mt_origins( + st: State, + locale: Locale, + user: crate::auth::MaybeUser, + name: Path, + q: Query, +) -> Result { + origins(st, locale, user, name, q, DnaType::MtDna).await +} + +/// The ancestral-origin icicle for one clade: the same top-down, containment-carries-descent shape +/// as the Big Tree, with each branch filled by where its men's most distant known ancestors came +/// from. See `proposals/ancestral-origin-icicle.md`. +async fn origins( + State(st): State, + locale: Locale, + user: crate::auth::MaybeUser, + Path(name): Path, + Query(q): Query, + dna_type: DnaType, +) -> Result { + let base_path = base_path_for(dna_type); + let node = du_db::haplogroup::get_by_name(&st.pool, &name, dna_type) + .await? + .ok_or_else(|| AppError::NotFound(format!("no haplogroup {name}")))?; + let level = origins_level(q.level.as_deref()); + let crumbs = build_crumbs(&st.pool, dna_type, base_path, &name).await?; + + // The era gate. A clade older than the ceiling gets its eligible children rather than a chart + // whose every band would read "Europe" — the reader is sent down, not turned away. + let too_old = node.tmrca_ybp.is_none_or(|t| t > ORIGINS_MAX_YBP); + if too_old { + let window = du_db::haplogroup::subtree_window(&st.pool, dna_type, &name, ORIGINS_DEPTH).await?; + let eligible: Vec = window + .iter() + .filter(|n| n.id != node.id.0 && n.tmrca_ybp.is_some_and(|t| t <= ORIGINS_MAX_YBP)) + .filter(|n| !is_private_node(&n.name)) + .map(|n| Crumb { + href: format!("{base_path}/node/{}/origins", encode(&n.name)), + name: n.name.clone(), + }) + .collect(); + let page = OriginsPageTemplate { + t: locale.t, + next: locale.next, + user: user.nav(), + base_path, + name, + level: level_code(level), + tmrca_ybp: node.tmrca_ybp, + max_ybp: ORIGINS_MAX_YBP, + crumbs, + laid: None, + eligible, + placed: 0, + }; + return Ok(html(&page)); + } + + // The whole subtree, not a display window: `origins_layout` prunes to the branches that + // actually carry an origin, so the drawn depth follows the data instead of a fixed number — + // and no sample is lost for sitting below an arbitrary cut-off. + let window = du_db::haplogroup::subtree_window(&st.pool, dna_type, &name, ORIGINS_DEPTH).await?; + // De-novo auto-named nodes must not surface publicly here any more than on the tree itself. + // They stay in the input marked `hidden`, so the ancestor walk is unbroken and their men are + // attributed to the nearest named branch rather than dropped. + let nodes: Vec = window + .iter() + .map(|n| origins_layout::Node { + id: n.id, + name: n.name.clone(), + parent_id: n.parent_id, + formed_ybp: n.formed_ybp, + tmrca_ybp: n.tmrca_ybp, + hidden: is_private_node(&n.name) || is_uuid_label(&n.name), + }) + .collect(); + + let published = du_db::origins::origins_under(&st.pool, dna_type, &name).await?; + let placed = du_db::origins::placed_under(&st.pool, &name, dna_type).await?; + let laid = origins_layout::layout(&nodes, &published, level, placed.max(0) as usize); + + Ok(html(&OriginsPageTemplate { + t: locale.t, + next: locale.next, + user: user.nav(), + base_path, + name, + level: level_code(level), + tmrca_ybp: node.tmrca_ybp, + max_ybp: ORIGINS_MAX_YBP, + crumbs, + laid: Some(laid), + eligible: Vec::new(), + placed: placed.max(0), + })) +} + +fn level_code(l: place::Level) -> &'static str { + match l { + place::Level::Country => "country", + place::Level::Admin => "admin", + place::Level::Locality => "place", + } +} + +fn encode(s: &str) -> String { + utf8_percent_encode(s, NON_ALPHANUMERIC).to_string() +} + +#[derive(askama::Template)] +#[template(path = "tree/origins.html")] +struct OriginsPageTemplate { + t: T, + next: String, + user: Option, + base_path: &'static str, + name: String, + level: &'static str, + tmrca_ybp: Option, + max_ybp: i32, + crumbs: Vec, + /// `None` when the clade is older than the era gate — the template shows `eligible` instead. + laid: Option, + /// Descendant clades that *are* inside the era, offered when this one is not. + eligible: Vec, + /// Placed samples under the clade — the denominator the composition is reported against. + placed: i64, +} + /// The "Geography & Time" panel fragment: a Leaflet map target (client fetches the /// GeoJSON above) plus a server-rendered SVG time axis of the clade's TMRCA/CI with any /// dated ancient-DNA anchors in the subtree. diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html new file mode 100644 index 00000000..93c293d2 --- /dev/null +++ b/rust/crates/du-web/templates/tree/origins.html @@ -0,0 +1,176 @@ +{% extends "base.html" %} +{% block title %}{{ t.get("tree.origins.title") }} · {{ name }} — {{ t.get("app.name") }}{% endblock %} +{# Full viewport width — the icicle wants all the horizontal room it can get. #} +{% block main_container %}container-fluid{% endblock %} +{% block content %} +{# + The ancestral-origin icicle (proposals/ancestral-origin-icicle.md §5). Time runs DOWN the page + on one absolute axis: a band's top is its branch's formation, its bottom is its TMRCA, so the + band's height IS its duration and two branches at the same date draw level wherever they sit in + the tree. Children are flush beneath their parent, so containment carries descent. + + Colour: the validated 8-slot categorical palette, fixed order, never cycled — a ninth locality + folds into the reserved neutral rather than getting an invented hue. Slot 0 is that neutral and + also carries "no locality recorded", because an absence is not an identity. +#} +
+

{{ t.get("tree.origins.title") }} · {{ name }}

+ + {# Level selector. Switching level changes WHICH entities exist, so re-ranking colours here is + not a repaint of survivors. #} +
+ {{ t.get("tree.origins.level.country") }} + {{ t.get("tree.origins.level.admin") }} + {{ t.get("tree.origins.level.place") }} +
+ + {{ t.get("tree.origins.back") }} +
+ + + +{% if let Some(l) = laid %} + +{# What the chart could not account for, stated before the chart rather than after it. #} +

+ {{ t.get("tree.origins.tmrca") }} + {% if let Some(ty) = tmrca_ybp %}{{ ty }} {{ t.get("tree.geo.axis") }}{% endif %} + · {{ placed }} {{ t.get("tree.origins.placed") }} + {% if l.unresolved > 0 %} + · {{ l.unresolved }} {{ t.get("tree.origins.unresolved") }} + {% endif %} + {% if l.pruned > 0 %} + · {{ l.pruned }} {{ t.get("tree.origins.pruned") }} + {% endif %} +

+ +{% if l.legend.len() >= 2 %} +{# A legend is always present for two or more series — identity is never colour alone. #} +
+ {% for e in l.legend %} + + + {% if let Some(lbl) = e.label %}{{ lbl }}{% else %}{{ t.get("tree.origins.unknown") }}{% endif %} + {{ e.count }} + + {% endfor %} +
+{% endif %} + +
+ + + {# An unmeasured branch is hatched so it can never read as a measured short one. #} + + + + + + + {# Time ruler: calendar years, one linear scale for the whole canvas. #} + {% for tk in l.ticks %} + + + {{ tk.label }} + + {% endfor %} + + {% for b in l.bands %} + + + + {% if !b.dated %} + + {% endif %} + {# Stacked composition. Segments are separated by surface, not by a stroke. #} + {% for s in b.segments %} + + {% if let Some(lbl) = s.label %}{{ lbl }}{% else %}{{ t.get("tree.origins.other") }}{% endif %} · {{ s.count }} + + {% endfor %} + {{ b.name }} — {{ b.with_origin }} {{ t.get("tree.origins.with") }}, {{ b.without_origin }} {{ t.get("tree.origins.without") }} + + {# Direct label: the relief the light-mode contrast warning obliges. Text wears text tokens, + never the series colour. #} + {% if !b.cramped %} + {{ b.name }} + {% endif %} + + {% endfor %} + + {# The men, as leaves on the present-day line. #} + {% for tp in l.tips %} + + + {{ tp.label }} + + {% endfor %} + +
+ +{# The table view — the accessibility fallback, and the relief for the light-mode contrast + warning. Everything the chart encodes in colour is readable here as text. #} +
+ {{ t.get("tree.origins.table") }} +
+ + + + + + + + + {% for e in l.legend %} + + + + + {% endfor %} + +
{{ t.get("tree.origins.col.locality") }}{{ t.get("tree.origins.col.count") }}
{% if let Some(lbl) = e.label %}{{ lbl }}{% else %}{{ t.get("tree.origins.unknown") }}{% endif %}{{ e.count }}
+
+
+ +{% else %} + +{# Older than the era gate: a signpost down, not a refusal. #} +
+

+ {{ t.get("tree.origins.tooold") }} + {% if let Some(ty) = tmrca_ybp %}({{ ty }}{% else %}({{ t.get("tree.geo.noage") }}{% endif %} + {{ t.get("tree.geo.axis") }}; {{ t.get("tree.origins.ceiling") }} {{ max_ybp }}) +

+ {% if eligible.is_empty() %} +

{{ t.get("tree.origins.nonebelow") }}

+ {% else %} +

{{ t.get("tree.origins.trybelow") }}

+
+ {% for c in eligible %} + {{ c.name }} + {% endfor %} +
+ {% endif %} +
+ +{% endif %} +{% endblock %} diff --git a/rust/locales/en.txt b/rust/locales/en.txt index 68841b0c..7f5db16a 100644 --- a/rust/locales/en.txt +++ b/rust/locales/en.txt @@ -168,6 +168,29 @@ tree.geo.nocoords=No sample coordinates available for this clade. tree.geo.noage=No age estimate yet for this clade. tree.geo.axis=years before present tree.geo.loading=Loading map… + +# Ancestral-origin icicle (proposals/ancestral-origin-icicle.md) +tree.origins.title=Ancestral origins +tree.origins.level=Detail level +tree.origins.level.country=Country +tree.origins.level.admin=County / State +tree.origins.level.place=Place +tree.origins.back=Back to tree +tree.origins.tmrca=Clade TMRCA +tree.origins.placed=placed samples +tree.origins.unresolved=with no published origin +tree.origins.pruned=branches hidden (no published origin below them) +tree.origins.with=with an origin +tree.origins.without=without one +tree.origins.unknown=No locality recorded +tree.origins.other=Other +tree.origins.table=Show the data as a table +tree.origins.col.locality=Locality +tree.origins.col.count=Samples +tree.origins.tooold=Ancestral origins are only meaningful in the genealogical era. This clade is older than that, so its branches would each aggregate to a whole continent. +tree.origins.ceiling=cutoff +tree.origins.trybelow=Try one of these younger branches below it: +tree.origins.nonebelow=No branch below this clade is young enough yet. tree.geo.tmrca=TMRCA tree.geo.formed=Formed tree.geo.legend.modern=Modern diff --git a/rust/locales/es.txt b/rust/locales/es.txt index 60e4aded..d240b1d2 100644 --- a/rust/locales/es.txt +++ b/rust/locales/es.txt @@ -120,6 +120,29 @@ tree.geo.nocoords=No hay coordenadas de muestras disponibles para este clado. tree.geo.noage=Aún no hay estimación de edad para este clado. tree.geo.axis=años antes del presente tree.geo.loading=Cargando mapa… + +# Ancestral-origin icicle (proposals/ancestral-origin-icicle.md) +tree.origins.title=Orígenes ancestrales +tree.origins.level=Nivel de detalle +tree.origins.level.country=País +tree.origins.level.admin=Condado / Estado +tree.origins.level.place=Localidad +tree.origins.back=Volver al árbol +tree.origins.tmrca=TMRCA del clado +tree.origins.placed=muestras situadas +tree.origins.unresolved=sin origen publicado +tree.origins.pruned=ramas ocultas (sin origen publicado debajo) +tree.origins.with=con origen +tree.origins.without=sin origen +tree.origins.unknown=Sin localidad registrada +tree.origins.other=Otros +tree.origins.table=Ver los datos como tabla +tree.origins.col.locality=Localidad +tree.origins.col.count=Muestras +tree.origins.tooold=Los orígenes ancestrales solo tienen sentido en la era genealógica. Este clado es más antiguo, así que cada una de sus ramas se agregaría a un continente entero. +tree.origins.ceiling=límite +tree.origins.trybelow=Pruebe una de estas ramas más recientes: +tree.origins.nonebelow=Todavía no hay ninguna rama lo bastante reciente bajo este clado. tree.geo.tmrca=TMRCA tree.geo.formed=Formado tree.geo.legend.modern=Moderno diff --git a/rust/locales/fr.txt b/rust/locales/fr.txt index 33a7744b..551f792a 100644 --- a/rust/locales/fr.txt +++ b/rust/locales/fr.txt @@ -120,6 +120,29 @@ tree.geo.nocoords=Aucune coordonnée d’échantillon disponible pour ce clade. tree.geo.noage=Pas encore d’estimation d’âge pour ce clade. tree.geo.axis=années avant le présent tree.geo.loading=Chargement de la carte… + +# Ancestral-origin icicle (proposals/ancestral-origin-icicle.md) +tree.origins.title=Origines ancestrales +tree.origins.level=Niveau de détail +tree.origins.level.country=Pays +tree.origins.level.admin=Comté / État +tree.origins.level.place=Lieu +tree.origins.back=Retour à l'arbre +tree.origins.tmrca=TMRCA du clade +tree.origins.placed=échantillons placés +tree.origins.unresolved=sans origine publiée +tree.origins.pruned=branches masquées (aucune origine publiée en dessous) +tree.origins.with=avec une origine +tree.origins.without=sans origine +tree.origins.unknown=Aucune localité enregistrée +tree.origins.other=Autres +tree.origins.table=Afficher les données sous forme de tableau +tree.origins.col.locality=Localité +tree.origins.col.count=Échantillons +tree.origins.tooold=Les origines ancestrales n'ont de sens que dans l'ère généalogique. Ce clade est plus ancien : chacune de ses branches se ramènerait à un continent entier. +tree.origins.ceiling=seuil +tree.origins.trybelow=Essayez l'une de ces branches plus récentes : +tree.origins.nonebelow=Aucune branche sous ce clade n'est encore assez récente. tree.geo.tmrca=TMRCA tree.geo.formed=Formé tree.geo.legend.modern=Moderne diff --git a/rust/migrations/0074_ancestral_origin.sql b/rust/migrations/0074_ancestral_origin.sql new file mode 100644 index 00000000..5df04b55 --- /dev/null +++ b/rust/migrations/0074_ancestral_origin.sql @@ -0,0 +1,69 @@ +-- Mirrored ancestral-origin records (`com.decodingus.atmosphere.ancestralOrigin`) — the +-- locality substrate for the genealogical-era origins icicle on the public tree. +-- Design: `proposals/ancestral-origin-icicle.md`. +-- +-- WHY THIS EXISTS. The AppView's only locality datum is `core.specimen_donor.geocoord`, and +-- of 9,642 placed Y samples only 1,380 carry one — all of them ancient or academic. The 7,882 +-- `cohort=bigy` D2C tips, which are the entire genealogical era, have 3 between them. A view of +-- where a branch's lines went had nothing to draw from. +-- +-- PRIVACY. This is the MDKA (most distant known ancestor) of a lineage: the surname, origin and +-- dates of the earliest documented paternal-line ancestor. Per +-- `proposals/biosample-identifier-dedup.md`, MDKA is genealogical context, NOT living-donor PII — +-- and this table narrows that rather than widening it. Gates enforced at ingest, each REJECTING +-- the record rather than merely hiding it (see `du_jobs::jetstream::build_ancestral_origin`): +-- +-- 1. `surname` is a single token — never a given name, whatever the client sent. +-- 2. `birth_year <= 1900`, the check that makes "not PII" verifiable rather than asserted. +-- 3. No birth year → country only; place text and coordinate are dropped. +-- 4. Coordinates coarsened to 2dp (~1 km) at ingest, because the publisher cannot be trusted +-- to have done it. +-- 5. The join key (an FTDNA kit id) is never rendered — every vendor namespace is +-- `core.biosample_identifier.is_public = false`. +-- +-- Two older migration headers say MDKA never reaches the AppView: `0012_fed_reporting` here and +-- `0030_mdka` in Navigator. They are NOT edited — both repos run `sqlx::migrate!`, which +-- checksums applied migrations, so editing even a comment would fail every existing database +-- with VersionMismatch. `proposals/ancestral-origin-icicle.md` §2 is the amendment of record. +-- +-- The D4 assertion-store PII rail (`research.assertion` rejecting MDKA_IS) STANDS UNCHANGED: it +-- governs assertions about a *living research subject* inside a project, which is a different +-- question from publishing a deceased ancestor's parish — and that rail is what keeps them apart. +-- +-- Envelope matches `fed.private_variant` (mig 0028): keyed (did, rkey), one collection per table, +-- idempotent time_us-ordered upsert from the firehose. + +CREATE TABLE fed.ancestral_origin ( + did TEXT NOT NULL, + rkey TEXT NOT NULL, + at_uri TEXT NOT NULL, + cid TEXT, + -- at-uri of the parent biosample record. Present only for genuinely federated samples; + -- the bulk-loaded tips that carry the genealogical era have none, so resolution normally + -- runs through `external_ids` below (see the design doc §4). + biosample_ref TEXT, + -- Published external identifiers, `[{namespace, value}]` — the working join key against + -- `core.biosample_identifier (namespace, value)`. Vendor namespaces stay background-only. + external_ids JSONB NOT NULL DEFAULT '[]'::jsonb, + lineage TEXT, -- Y_DNA | MT_DNA + surname TEXT, -- single token; gate 1 + origin_place TEXT, -- as recorded; normalized at read time by du_db::place + origin_country TEXT, + birth_year INTEGER, -- gate 2 bounds this + death_year INTEGER, + geocoord geometry(Point, 4326), -- gate 4 coarsens this + record_created_at TIMESTAMPTZ, + time_us BIGINT NOT NULL, + indexed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (did, rkey) +); + +-- Resolution paths: the at-uri join (federated samples) and the identifier join (everything on +-- the tree today). The GIN index serves the `external_ids @> [{namespace,value}]` containment +-- lookup the aggregate uses. +CREATE INDEX fed_ancestral_origin_biosample_idx ON fed.ancestral_origin (biosample_ref) + WHERE biosample_ref IS NOT NULL; +CREATE INDEX fed_ancestral_origin_extids_gin ON fed.ancestral_origin + USING gin (external_ids jsonb_path_ops); +-- The icicle reads one arm at a time. +CREATE INDEX fed_ancestral_origin_lineage_idx ON fed.ancestral_origin (lineage); From 15f03d4537ba425e4fde3fbee0a2040f5ea1c26b Mon Sep 17 00:00:00 2001 From: James Kane Date: Thu, 6 Aug 2026 10:52:09 -0500 Subject: [PATCH 04/10] feat(tree): preview data for the origins icicle, and fit labels to their boxes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The icicle ships dark — `import_kit_identifiers.rs` reserves MDKA for records a PDS publishes, and the Navigator publisher is not built — so there was no honest way to put the view in front of a reviewer. `seed-ancestral-origins.sql` fabricates a plausible cohort against REAL tree placements: 708 rows over three era-gated clades (R-DF85, R-S764, R-Z3000), every row stamped `did = 'did:plc:preview'` so the set removes with one delete and can never be mistaken for a contributor's record. The mix is deliberately awkward rather than tidy, because the tidy version hides the cases that matter: more than eight distinct counties (so the fold into the reserved "Other" slot is exercised), US diaspora beside Irish counties, country-only rows, and men with no locality at all. Rows are written already conformant to the ingest gates — single-token surnames, birth years at or before 1900, 2dp coordinates, and no place or coordinate where there is no birth year — so the preview shows what ingest would actually have kept rather than a state it would have rejected. Seeding it immediately exposed a bug that the layout tests could not: the boxes are sized by the phylogeny, not by the text. A tip is at most 74px and holds about 13 characters, while `Sullivan · Co. Limerick` is 23, so labels ran straight through their neighbours and the tip row was unreadable. `fit()` now truncates band and tip labels to their box with an ellipsis, and the unabbreviated form moves to the hover title so nothing is lost. The rectangle assertions all passed throughout — this was only ever visible by rendering it. Also recorded from the same run, unfixed: with dense data pruning stops helping. R-DF85 draws 266 bands across 11,220px because 278 of its 283 samples carry an origin, and the page scrolls horizontally like the Big Tree and FTDNA's block tree do. That is inherent to one box per man at a legible width, not a defect, but it is the shape a width-reduction pass would target. Co-Authored-By: Claude Opus 5 (1M context) --- rust/crates/du-web/src/origins_layout.rs | 52 ++++++- .../crates/du-web/templates/tree/origins.html | 4 +- rust/scripts/seed-ancestral-origins.sql | 142 ++++++++++++++++++ 3 files changed, 195 insertions(+), 3 deletions(-) create mode 100644 rust/scripts/seed-ancestral-origins.sql diff --git a/rust/crates/du-web/src/origins_layout.rs b/rust/crates/du-web/src/origins_layout.rs index 72f79556..699ec9ea 100644 --- a/rust/crates/du-web/src/origins_layout.rs +++ b/rust/crates/du-web/src/origins_layout.rs @@ -87,12 +87,16 @@ pub struct Band { pub segments: Vec, /// True when the band is too short to letter — the view puts its label in the tooltip only. pub cramped: bool, + /// `name` fitted to the band's width. The full name is always in the band's ``. + pub label: String, } /// One man, as a leaf below the branch he is placed on. #[derive(Debug, Clone, PartialEq)] pub struct Tip { + /// Fitted to the box. The unabbreviated form is [`Self::full`], shown on hover. pub label: String, + pub full: String, pub slot: usize, pub x: f64, pub y: f64, @@ -424,6 +428,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl } bands.push(Band { id: n.id, + label: fit(&n.name, extent[i], 10.0), name: n.name.clone(), x: left[i], y, @@ -470,7 +475,8 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl }; tips.push(Tip { slot: locality.and_then(|l| slots.get(l).copied()).unwrap_or(0), - label, + full: label.clone(), + label: fit(&label, w, 9.0), x: bx + k as f64 * (w + H_GAP), y: tip_y, w, @@ -496,6 +502,30 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl } } +/// Approximate width of one character of the SVG label font, as a fraction of its size. The +/// canvas has no text metrics, so labels are fitted arithmetically; erring narrow would clip text +/// that fits, erring wide lets it spill. +const CHAR_W_RATIO: f64 = 0.55; + +/// Truncate a label to what actually fits in `width` at `font_px`, with an ellipsis. +/// +/// Necessary because the boxes are sized by the phylogeny, not by the text: a tip is at most +/// [`LEAF_W`] wide and holds ~13 characters, while `Sullivan · Co. Limerick` is 23. Left +/// unfitted, labels ran straight through their neighbours and the tip row became unreadable — +/// visible immediately on a real clade, invisible to a layout test that only checks rectangles. +fn fit(label: &str, width: f64, font_px: f64) -> String { + let per_char = font_px * CHAR_W_RATIO; + let budget = ((width - 4.0) / per_char).floor().max(0.0) as usize; + let chars: Vec<char> = label.chars().collect(); + if chars.len() <= budget { + return label.to_string(); + } + if budget <= 1 { + return String::new(); + } + chars[..budget - 1].iter().collect::<String>().trim_end().to_string() + "…" +} + /// Children before parents, iteratively — the tree is user-shaped and may be deep enough to blow /// a recursive stack. fn post_order(children: &[Vec<usize>], root: usize) -> Vec<usize> { @@ -781,6 +811,26 @@ mod tests { assert!(laid.legend.iter().any(|e| e.label.is_none() && e.slot == 0)); } + /// Boxes are sized by the phylogeny, not by the text, so labels must be cut to fit. Seen on a + /// real clade: `Grant · United States` ran straight through its neighbours and the tip row + /// was unreadable — which the rectangle-only layout assertions could never have caught. + #[test] + fn labels_are_fitted_to_their_boxes() { + assert_eq!(fit("Kane", 74.0, 9.0), "Kane", "what fits is left alone"); + let cut = fit("Sullivan · Co. Limerick", 74.0, 9.0); + assert!(cut.ends_with('…') && cut.chars().count() < 23); + assert!(fit("anything", 4.0, 9.0).is_empty(), "no room at all yields no text"); + + let laid = layout(&tree(), &[origin(2, "Kenmare, Co. Kerry, Ireland")], Level::Admin, 1); + let tip = laid.tips.first().expect("one man"); + assert_eq!(tip.full, "Kane · Co. Kerry", "the full label survives for the tooltip"); + assert!(tip.label.chars().count() <= tip.full.chars().count()); + // Every band's drawn label fits the band it sits in. + for b in &laid.bands { + assert!((b.label.chars().count() as f64) * 10.0 * CHAR_W_RATIO <= b.w, "{}", b.name); + } + } + #[test] fn an_empty_window_lays_out_to_nothing_rather_than_panicking() { assert_eq!(layout(&[], &[], Level::Country, 0), Laid::default()); diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html index 93c293d2..7684fb03 100644 --- a/rust/crates/du-web/templates/tree/origins.html +++ b/rust/crates/du-web/templates/tree/origins.html @@ -111,7 +111,7 @@ <h1 class="h3 mb-0 me-auto">{{ t.get("tree.origins.title") }} · {{ name }}</h1> {# Direct label: the relief the light-mode contrast warning obliges. Text wears text tokens, never the series colour. #} {% if !b.cramped %} - <text x="{{ b.x }}" y="{{ b.y }}" dx="4" dy="11" font-size="10" class="origins-band-label">{{ b.name }}</text> + <text x="{{ b.x }}" y="{{ b.y }}" dx="4" dy="11" font-size="10" class="origins-band-label">{{ b.label }}</text> {% endif %} </g> {% endfor %} @@ -120,7 +120,7 @@ <h1 class="h3 mb-0 me-auto">{{ t.get("tree.origins.title") }} · {{ name }}</h1> {% for tp in l.tips %} <g class="origins-tip"> <rect x="{{ tp.x }}" y="{{ tp.y }}" width="{{ tp.w }}" height="{{ tp.h }}" - class="origins-seg s{{ tp.slot }}" rx="2"></rect> + class="origins-seg s{{ tp.slot }}" rx="2"><title>{{ tp.full }} {{ tp.label }} {% endfor %} diff --git a/rust/scripts/seed-ancestral-origins.sql b/rust/scripts/seed-ancestral-origins.sql new file mode 100644 index 00000000..b816de93 --- /dev/null +++ b/rust/scripts/seed-ancestral-origins.sql @@ -0,0 +1,142 @@ +-- PREVIEW DATA ONLY — synthetic ancestral origins for exercising the origins icicle +-- (`/ytree/node/:name/origins`, proposals/ancestral-origin-icicle.md) on a local database. +-- +-- psql "$DATABASE_URL" -f scripts/seed-ancestral-origins.sql +-- psql "$DATABASE_URL" -c "DELETE FROM fed.ancestral_origin WHERE did = 'did:plc:preview';" +-- +-- NEVER run this against production. Every row is stamped `did = 'did:plc:preview'` so the +-- whole set is removable with the one-line delete above, and so nothing here can be mistaken +-- for a real contributor's record. +-- +-- WHY IT EXISTS. The view ships dark: `import_kit_identifiers.rs` reserves MDKA for records a +-- PDS publishes, and the Navigator publisher is not built yet, so there is no honest way to get +-- data in front of a reviewer. This fabricates a plausible cohort against REAL tree placements +-- so the layout, the palette, the era gate and the pruning can be seen working. +-- +-- WHAT IT WRITES. Rows are inserted straight into the mirror, bypassing the Jetstream consumer +-- where the privacy gates live — so every row here is written already conformant to them, and +-- the preview shows what ingest would actually have kept: +-- +-- * `surname` is one token (the gate rejects a given name); +-- * `birth_year` is always <= 1900; +-- * a row with NO birth year carries a country and NOTHING finer — that is the §2.3 +-- precision ladder, not an oversight; +-- * coordinates are 2dp. +-- +-- The mix is deliberately awkward, so the preview shows the hard cases rather than a tidy one: +-- more than eight distinct counties (exercises the fold into the reserved "Other" slot), US +-- diaspora alongside Irish counties, country-only rows, and men with no locality at all. + +BEGIN; + +-- The clades to populate: era-gated (TMRCA under the 1500 ybp ceiling) and deep enough in kits +-- to show real branching. Add or swap names here to preview a different part of the tree. +CREATE TEMP TABLE preview_root(name TEXT) ON COMMIT DROP; +INSERT INTO preview_root(name) VALUES ('R-DF85'), ('R-S764'), ('R-Z3000'); + +-- 40 slots, cycled over the kits in a stable order. Proportions are roughly an Irish surname +-- project's: a long Munster tail, a Scottish and English minority, a US diaspora, and a real +-- fraction with nothing recorded. +CREATE TEMP TABLE preview_mix( + slot INT, surname TEXT, place TEXT, country TEXT, byear INT, lat FLOAT8, lon FLOAT8 +) ON COMMIT DROP; +INSERT INTO preview_mix VALUES + ( 0,'Sullivan','Kenmare, Co. Kerry, Ireland','Ireland',1812, 51.88, -9.58), + ( 1,'McCarthy','Bandon, Co. Cork, Ireland','Ireland',1799, 51.75, -8.74), + ( 2,'Donovan','Skibbereen, Co. Cork, Ireland','Ireland',1855, 51.55, -9.26), + ( 3,'Kane','Creegh South, Co. Clare, Ireland','Ireland',1830, 52.75, -9.43), + ( 4,'Murphy','Cork, Co. Cork, Ireland','Ireland',1841, 51.90, -8.47), + ( 5,'Sullivan','Cahersiveen, Co. Kerry, Ireland','Ireland',1826, 51.95, -10.22), + ( 6,'Brien','Ennis, Co. Clare, Ireland','Ireland',1808, 52.84, -8.99), + ( 7,'Walsh','Clonmel, Co. Tipperary, Ireland','Ireland',1863, 52.35, -7.70), + ( 8,'Ryan','Nenagh, Co. Tipperary, Ireland','Ireland',1834, 52.86, -8.20), + ( 9,'McCarthy','Macroom, Co. Cork, Ireland','Ireland',1798, 51.90, -8.96), + (10,'Connor','Galway, Co. Galway, Ireland','Ireland',1849, 53.27, -9.05), + (11,'Fitzgerald','Dungarvan, Co. Waterford, Ireland','Ireland',1817, 52.09, -7.62), + (12,'Kelly','Westport, Co. Mayo, Ireland','Ireland',1852, 53.80, -9.52), + (13,'Power','Kilkenny, Co. Kilkenny, Ireland','Ireland',1805, 52.65, -7.25), + (14,'Barry','Wexford, Co. Wexford, Ireland','Ireland',1868, 52.34, -6.46), + (15,'Sullivan','Limerick, Co. Limerick, Ireland','Ireland',1821, 52.66, -8.63), + (16,'Doyle','Adare, Co. Limerick, Ireland','Ireland',1839, 52.56, -8.79), + (17,'Cronin','Killarney, Co. Kerry, Ireland','Ireland',1811, 52.06, -9.51), + -- Scotland and England: the constituent countries must stay distinct from "United Kingdom". + (18,'Kelly','Moulin, Pitlochry PH16 5EP, UK','Scotland',1820, 56.71, -3.74), + (19,'MacLeod','Stornoway, Isle of Lewis, Scotland','Scotland',1844, 58.21, -6.39), + (20,'Campbell','Oban, Argyll, Scotland','Scotland',1802, 56.41, -5.47), + (21,'Hughes','Chelmsford, England, UK','England',1858, 51.73, 0.48), + (22,'Ward','Liverpool, England, UK','England',1836, 53.41, -2.98), + -- Diaspora, as recorded. No inference is made about where these lines "really" came from. + (23,'Brazil','Pickens County, SC, USA','United States',1801, 34.88, -82.71), + (24,'ODonnell','Amelia County, VA 23002, USA','United States',1788, 37.34, -77.98), + (25,'Sullivan','Boston, MA 02108, USA','United States',1847, 42.36, -71.06), + (26,'Murphy','Wythe County, VA, USA','United States',1793, 36.92, -81.08), + (27,'Kane','Hinds County, MS, USA','United States',1866, 32.26, -90.36), + (28,'Walsh','Toronto, ON, Canada','Canada',1859, 43.65, -79.38), + (29,'Ryan','Sydney, NSW, Australia','Australia',1854,-33.87, 151.21), + -- Country only: no birth year, so the precision ladder withholds place and coordinate. + (30,'Collins',NULL,'Ireland',NULL,NULL,NULL), + (31,'Nolan',NULL,'Ireland',NULL,NULL,NULL), + (32,'Moore',NULL,'Scotland',NULL,NULL,NULL), + (33,'Grant',NULL,'United States',NULL,NULL,NULL), + -- Published, but with no locality at all — must draw as its own visible slice. + (34,'Quinn',NULL,NULL,NULL,NULL,NULL), + (35,'Byrne',NULL,NULL,NULL,NULL,NULL), + (36,'Flynn',NULL,NULL,NULL,NULL,NULL), + -- A few more counties, pushing the distinct count past the eight palette slots. + (37,'Brennan','Sligo, Co. Sligo, Ireland','Ireland',1828, 54.27, -8.48), + (38,'Duffy','Letterkenny, Co. Donegal, Ireland','Ireland',1815, 54.95, -7.73), + (39,'Reilly','Cavan, Co. Cavan, Ireland','Ireland',1871, 53.99, -7.36); + +-- Every FTDNA-identified placed Y sample under the preview roots, in a stable order so a re-run +-- assigns the same locality to the same kit. +WITH RECURSIVE sub AS ( + SELECT h.id, h.name AS root + FROM tree.haplogroup h JOIN preview_root p ON p.name = h.name + WHERE h.haplogroup_type = 'Y_DNA' AND h.valid_until IS NULL + UNION ALL + SELECT r.child_haplogroup_id, s.root + FROM sub s JOIN tree.haplogroup_relationship r + ON r.parent_haplogroup_id = s.id AND r.valid_until IS NULL +), +kits AS ( + SELECT DISTINCT ON (i.value) i.value AS kit + FROM tree.haplogroup_sample hs + JOIN sub ON sub.id = hs.haplogroup_id + JOIN core.biosample b ON b.sample_guid = hs.sample_guid AND b.deleted = false + JOIN core.biosample_identifier i ON i.sample_guid = hs.sample_guid AND i.namespace = 'FTDNA' + WHERE hs.dna_type = 'Y_DNA' AND hs.status IN ('PLACED','CURATED') + ORDER BY i.value +), +numbered AS (SELECT kit, (row_number() OVER (ORDER BY kit) - 1) AS rn FROM kits) +INSERT INTO fed.ancestral_origin + (did, rkey, at_uri, external_ids, lineage, surname, origin_place, origin_country, + birth_year, death_year, geocoord, record_created_at, time_us) +SELECT + 'did:plc:preview', + 'kit-' || n.kit, + 'at://did:plc:preview/com.decodingus.atmosphere.ancestralOrigin/kit-' || n.kit, + jsonb_build_array(jsonb_build_object('namespace','FTDNA','value', n.kit)), + 'Y_DNA', + m.surname, + m.place, + m.country, + m.byear, + -- A death year only where a birth year established the ancestor at all. + CASE WHEN m.byear IS NOT NULL THEN m.byear + 55 + (n.rn % 20) END, + CASE WHEN m.lat IS NOT NULL AND m.lon IS NOT NULL + THEN ST_SetSRID(ST_MakePoint(round(m.lon::numeric, 2), round(m.lat::numeric, 2)), 4326) END, + now(), + 1 +FROM numbered n +JOIN preview_mix m ON m.slot = n.rn % 40 +ON CONFLICT (did, rkey) DO NOTHING; + +COMMIT; + +-- What landed, and at what coverage. +SELECT count(*) AS preview_rows, + count(*) FILTER (WHERE origin_place IS NOT NULL) AS with_place, + count(*) FILTER (WHERE origin_country IS NOT NULL) AS with_country, + count(*) FILTER (WHERE geocoord IS NOT NULL) AS with_geocoord, + count(*) FILTER (WHERE birth_year IS NULL) AS no_birth_year +FROM fed.ancestral_origin WHERE did = 'did:plc:preview'; From 4ffc3b8260fbffc08c134ce495055f83f6187b23 Mon Sep 17 00:00:00 2001 From: James Kane Date: Thu, 6 Aug 2026 13:59:34 -0500 Subject: [PATCH 05/10] fix(tree): make the origins icicle readable on a real clade MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R-DF85 drew 266 branches across an 11,220px canvas — six screens of horizontal scrolling for a chart nobody could read. Canvas width is driven by the number of leaf branches, so pruning empty ones only helps where data is sparse; on a well-covered clade (278 of 283 samples carry an origin) it does nothing. Three changes, all found by looking at the rendered page rather than the tests. A DEPTH BOUND, defaulting to 4 levels with a selector (1-8). This is a legibility bound, not a data one: branches past the depth are marked `hidden` rather than dropped, which routes them through the same path that already handles de-novo nodes — their men are attributed to the nearest drawn ancestor. So every band's composition is identical at every depth and only the visible branching changes, which `folding_by_depth_preserves_composition_exactly` pins. R-DF85 goes 11,220px → 3,498px at the default, 2,016px at depth 2. Folded bands carry a "+" and say so on hover, so "this branch is simple" is never confused with "you are not being shown its shape". TIPS TOO NARROW TO LABEL ARE DROPPED AND COUNTED. Men share their band's width, so a band holding forty of them produced forty 8px slivers that hid the composition bar above them rather than adding anything. Below 26px the box goes and the man is counted instead — reported on the page, never silently. "NO LOCALITY RECORDED" IS NOW DRAWN. It was counted, legended, and then left as bare band background, so the chart disagreed with its own legend and a branch of unrecorded men looked like a branch with fewer men. It is now a real segment, always last so absence sits at the same end of every bar. That exposed a second problem: slot 0 carries both "Other" (localities past the eight palette slots) and "no locality recorded", and the legend called both of them the latter. They share a colour but not a meaning, so `Segment::unknown` now separates them in the legend, the tooltips and the table. Co-Authored-By: Claude Opus 5 (1M context) --- rust/crates/du-web/assets/main.css | 1 + rust/crates/du-web/src/origins_layout.rs | 195 ++++++++++++++++-- rust/crates/du-web/src/routes/tree.rs | 47 ++++- .../crates/du-web/templates/tree/origins.html | 36 +++- rust/locales/en.txt | 2 + rust/locales/es.txt | 2 + rust/locales/fr.txt | 2 + 7 files changed, 251 insertions(+), 34 deletions(-) diff --git a/rust/crates/du-web/assets/main.css b/rust/crates/du-web/assets/main.css index 9d7c94b1..ffb19469 100644 --- a/rust/crates/du-web/assets/main.css +++ b/rust/crates/du-web/assets/main.css @@ -256,3 +256,4 @@ code { color: #495057; background-color: #f8f9fa; } .origins-swatch.s6 { background: var(--o-6); } .origins-swatch.s7 { background: var(--o-7); } .origins-swatch.s8 { background: var(--o-8); } +.origins-more { fill: var(--bs-secondary-color, #6c757d); font-weight: 700; } diff --git a/rust/crates/du-web/src/origins_layout.rs b/rust/crates/du-web/src/origins_layout.rs index 699ec9ea..d63db8a1 100644 --- a/rust/crates/du-web/src/origins_layout.rs +++ b/rust/crates/du-web/src/origins_layout.rs @@ -44,6 +44,10 @@ const MIN_BAND_H: f64 = 18.0; const UNDATED_H: f64 = MIN_BAND_H; /// Sample tips hang in a band below the youngest branch. const TIP_H: f64 = 16.0; +/// Narrowest a man's box may be and still carry a readable label. Below it the box is dropped and +/// the man counted instead — a row of 8px slivers hides the composition bar rather than adding to +/// it. +const MIN_TIP_W: f64 = 26.0; const TIP_GAP: f64 = 10.0; const GUTTER_W: f64 = 54.0; const MARGIN: f64 = 8.0; @@ -63,6 +67,10 @@ pub struct Segment { pub label: Option, pub count: usize, pub slot: usize, + /// Distinguishes the two things slot 0 carries: `true` = "no locality recorded" (an absence), + /// `false` with no label = "Other" (localities past the palette). They share a colour but not + /// a meaning, so the legend and tooltips must not call both the same thing. + pub unknown: bool, pub x: f64, pub w: f64, } @@ -89,6 +97,10 @@ pub struct Band { pub cramped: bool, /// `name` fitted to the band's width. The full name is always in the band's ``. pub label: String, + /// Branches below this one were folded into it by the depth bound. Their men are counted in + /// this band's composition; their sub-branching is not drawn. The view marks these so a + /// reader can tell "this branch is simple" from "you are not being shown its shape". + pub has_more: bool, } /// One man, as a leaf below the branch he is placed on. @@ -120,6 +132,8 @@ pub struct LegendEntry { pub label: Option<String>, pub slot: usize, pub count: usize, + /// See [`Segment::unknown`]. + pub unknown: bool, } #[derive(Debug, Clone, Default, PartialEq)] @@ -135,6 +149,9 @@ pub struct Laid { /// Branches dropped because no origin sits beneath them. Reported, never silent: a pruned /// chart that looked complete would misrepresent how much of the clade this is. pub pruned: usize, + /// Men whose per-man box was too narrow to letter. They remain in their band's composition; + /// only the box is gone. + pub tips_suppressed: usize, } /// The minimum a node needs from the tree window. Mirrors `du_db::haplogroup::WindowNode` so this @@ -164,12 +181,24 @@ pub struct Node { /// and says nothing. On a real clade that was 175 bands and a 7,944px canvas for 10 origins. /// Pruning is reported, never silent. /// -/// Returns the retained nodes (root always kept), the origins re-pointed at visible branches, and -/// how many branches were pruned. -pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> (Vec<Node>, Vec<SampleOrigin>, usize) { +/// Retained nodes, origins re-pointed at visible branches, how many branches were pruned, and +/// which retained branches have folded descendants. +pub struct Pruned { + pub nodes: Vec<Node>, + pub origins: Vec<SampleOrigin>, + pub pruned: usize, + pub has_more: std::collections::HashSet<i64>, +} + +pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> Pruned { let by_id: HashMap<i64, &Node> = nodes.iter().map(|n| (n.id, n)).collect(); let Some(root) = nodes.iter().find(|n| n.parent_id.is_none()) else { - return (Vec::new(), Vec::new(), 0); + return Pruned { + nodes: Vec::new(), + origins: Vec::new(), + pruned: 0, + has_more: std::collections::HashSet::new(), + }; }; // Climb to the nearest visible ancestor. The root is the floor: it is always drawn, so no @@ -242,7 +271,28 @@ pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> (Vec<Node>, }) .collect(); let pruned = nodes.iter().filter(|n| !n.hidden).count() - retained.len(); - (retained, moved, pruned) + + // Which retained branches have folded descendants — every retained ancestor of a hidden node. + // Only *hidden* (depth-folded) nodes count: a branch pruned for carrying no origin adds + // nothing a reader could drill into. + let retained_ids: std::collections::HashSet<i64> = retained.iter().map(|n| n.id).collect(); + let mut has_more = std::collections::HashSet::new(); + for n in nodes.iter().filter(|n| n.hidden) { + let mut at = n.parent_id; + let mut guard = 0; + while let Some(id) = at { + if guard > nodes.len() { + break; + } + if retained_ids.contains(&id) { + has_more.insert(id); + break; + } + at = by_id.get(&id).and_then(|p| p.parent_id); + guard += 1; + } + } + Pruned { nodes: retained, origins: moved, pruned, has_more } } /// Roll each sample's locality up to its branch **and every ancestor of that branch**, so a band's @@ -327,13 +377,21 @@ fn segments_for( slot: slots[l], label: Some(l.clone()), count: n, + unknown: false, x: 0.0, w: 0.0, }) .collect(); let with_origin: usize = segs.iter().map(|s| s.count).sum::<usize>() + other; if other > 0 { - segs.push(Segment { label: None, count: other, slot: 0, x: 0.0, w: 0.0 }); + segs.push(Segment { label: None, count: other, slot: 0, unknown: false, x: 0.0, w: 0.0 }); + } + // "No locality recorded" is DRAWN, always last, so absence sits at the same end of every bar + // and bands can be compared by eye. Leaving it as bare background — which is what happened + // until a real clade was rendered — made the chart disagree with its own legend, and made a + // branch whose men are unrecorded look like a branch with fewer men. + if unknown > 0 { + segs.push(Segment { label: None, count: unknown, slot: 0, unknown: true, x: 0.0, w: 0.0 }); } (segs, with_origin, unknown) } @@ -343,8 +401,9 @@ fn segments_for( pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, placed_total: usize) -> Laid { // Attribute origins to visible branches and drop the branches with none beneath them, before // anything is measured — see `prune_to_origins`. - let (nodes, origins, pruned) = prune_to_origins(all_nodes, all_origins); - let (nodes, origins) = (&nodes[..], &origins[..]); + let p = prune_to_origins(all_nodes, all_origins); + let (pruned, has_more) = (p.pruned, p.has_more); + let (nodes, origins) = (&p.nodes[..], &p.origins[..]); let Some(root) = nodes.iter().find(|n| n.parent_id.is_none()) else { return Laid::default(); }; @@ -441,6 +500,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl without_origin, segments: segs, cramped: h < 14.0, + has_more: has_more.contains(&n.id), }); deepest = deepest.max(y + h); @@ -460,11 +520,20 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl per_node.entry(o.haplogroup_id).or_default().push(o); } let band_x: HashMap<i64, (f64, f64)> = bands.iter().map(|b| (b.id, (b.x, b.w))).collect(); + let mut tips_suppressed = 0usize; for (node_id, mut list) in per_node { let Some(&(bx, bw)) = band_x.get(&node_id) else { continue }; list.sort_by(|a, b| a.sample_guid.cmp(&b.sample_guid)); let n = list.len() as f64; - let w = ((bw - H_GAP * (n - 1.0).max(0.0)) / n).min(LEAF_W).max(8.0); + let w = (bw - H_GAP * (n - 1.0).max(0.0)) / n; + // Below this a tip is a coloured sliver with no legible label — noise that hides the + // composition bar above it. Those men are still counted in the band; only the per-man box + // is dropped, and the count is reported. + if w < MIN_TIP_W { + tips_suppressed += list.len(); + continue; + } + let w = w.min(LEAF_W); for (k, o) in list.iter().enumerate() { let locality = o.place.label_at(level); let label = match (&o.surname, locality) { @@ -499,6 +568,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl legend, unresolved: placed_total.saturating_sub(resolved), pruned, + tips_suppressed, } } @@ -572,15 +642,13 @@ fn era_label(ybp: i32) -> String { } fn legend_for(root_comp: &HashMap<Option<String>, usize>, slots: &HashMap<String, usize>) -> Vec<LegendEntry> { - let (segs, _, unknown) = segments_for(root_comp, slots); - let mut out: Vec<LegendEntry> = segs + // The legend is exactly the root band's segments — including the drawn "Other" and + // "no locality recorded" bars, which is what keeps chart and legend from disagreeing. + segments_for(root_comp, slots) + .0 .into_iter() - .map(|s| LegendEntry { label: s.label, slot: s.slot, count: s.count }) - .collect(); - if unknown > 0 { - out.push(LegendEntry { label: None, slot: 0, count: unknown }); - } - out + .map(|s| LegendEntry { label: s.label, slot: s.slot, count: s.count, unknown: s.unknown }) + .collect() } #[cfg(test)] @@ -649,7 +717,37 @@ mod tests { let laid = layout(&tree(), &origins, Level::Admin, 2); let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); assert_eq!(root.with_origin, 1); - assert_eq!(root.without_origin, 1, "drawn, never omitted"); + assert_eq!(root.without_origin, 1, "counted"); + + // And DRAWN — left as bare background it made the chart disagree with its own legend, and + // a branch of unrecorded men looked like a branch with fewer men. + let absent = root.segments.iter().find(|s| s.unknown).expect("an unknown segment exists"); + assert_eq!(absent.count, 1); + assert_eq!(absent.slot, 0, "an absence never wears a categorical hue"); + assert!(absent.w > 0.0, "it occupies real width"); + // Absence sits last in every bar, so bands can be compared by eye. + assert!(root.segments.last().unwrap().unknown); + // Segments now account for the whole band. + let covered: f64 = root.segments.iter().map(|s| s.w).sum::<f64>() + + SEG_GAP * (root.segments.len() - 1) as f64; + assert!((covered - root.w).abs() < 0.01, "the bar is fully accounted for"); + } + + /// Slot 0 carries two different things. They share a colour but not a meaning, and the legend + /// must not call both "no locality recorded". + #[test] + fn other_and_no_locality_are_distinguishable() { + let mut comp: HashMap<Option<String>, usize> = + (0..10).map(|i| (Some(format!("Place {i:02}")), 10 - i)).collect(); + comp.insert(None, 4); + let slots = assign_slots(&comp); + let (segs, _, _) = segments_for(&comp, &slots); + + let other = segs.iter().find(|s| s.slot == 0 && !s.unknown).expect("Other"); + let absent = segs.iter().find(|s| s.unknown).expect("no locality recorded"); + assert_eq!(absent.count, 4); + assert!(other.count > 0 && other.label.is_none()); + assert!(segs.last().unwrap().unknown, "absence is always last"); } /// Colour follows the entity. Ranking is done once over the root and held, so drilling into a @@ -867,6 +965,67 @@ mod tests { assert_eq!(root.with_origin, 1, "and still rolls up to the root"); } + /// The depth bound is a LEGIBILITY bound, not a data one. Folding a branch must move its men + /// into the nearest drawn ancestor, so the composition a reader sees is identical at every + /// depth — only the visible branching changes. R-DF85 drew 266 bands across 11,220px unbounded. + #[test] + fn folding_by_depth_preserves_composition_exactly() { + let deep = vec![ + node(1, "R-Root", None, Some(1600), Some(1400)), + node(2, "R-Mid", Some(1), Some(1400), Some(1100)), + node(3, "R-Deep", Some(2), Some(1100), Some(800)), + ]; + let origins = vec![ + origin(2, "Cork, Co. Cork, Ireland"), + origin(3, "Kenmare, Co. Kerry, Ireland"), + origin(3, "Bandon, Co. Cork, Ireland"), + ]; + let full = layout(&deep, &origins, Level::Admin, 3); + + // Fold everything below R-Mid, exactly as the route does past the display depth. + let mut folded_nodes = deep.clone(); + folded_nodes[2].hidden = true; + let folded = layout(&folded_nodes, &origins, Level::Admin, 3); + + let root_of = |l: &Laid| l.bands.iter().find(|b| b.id == 1).unwrap().clone(); + assert_eq!(root_of(&full).with_origin, root_of(&folded).with_origin, "3 men either way"); + let seg = |l: &Laid, id: i64, name: &str| { + l.bands + .iter() + .find(|b| b.id == id) + .unwrap() + .segments + .iter() + .find(|s| s.label.as_deref() == Some(name)) + .map(|s| s.count) + }; + assert_eq!(seg(&full, 1, "Co. Cork"), Some(2)); + assert_eq!(seg(&folded, 1, "Co. Cork"), Some(2), "unchanged by folding"); + assert_eq!(seg(&folded, 1, "Co. Kerry"), Some(1)); + // R-Deep is gone from the drawing, and its men are now R-Mid's. + assert!(folded.bands.iter().all(|b| b.id != 3)); + assert_eq!(folded.bands.iter().find(|b| b.id == 2).unwrap().with_origin, 3); + // And the fold is advertised, so "simple" is never confused with "not shown". + assert!(folded.bands.iter().find(|b| b.id == 2).unwrap().has_more); + assert!(!full.bands.iter().find(|b| b.id == 2).unwrap().has_more); + } + + /// A row of 8px slivers hides the composition bar instead of adding to it. The men stay + /// counted; only the per-man box goes, and the count is reported. + #[test] + fn tips_too_narrow_to_label_are_dropped_and_counted() { + let crowd: Vec<SampleOrigin> = (0..40).map(|_| origin(2, "Cork, Co. Cork, Ireland")).collect(); + let laid = layout(&tree(), &crowd, Level::Admin, 40); + assert!(laid.tips.is_empty(), "40 men cannot each hold a legible box"); + assert_eq!(laid.tips_suppressed, 40, "reported, never silent"); + // They are still fully present in the composition. + assert_eq!(laid.bands.iter().find(|b| b.id == 2).unwrap().with_origin, 40); + // Every tip that IS drawn is wide enough to letter. + let few = layout(&tree(), &[origin(2, "Cork, Co. Cork, Ireland")], Level::Admin, 1); + assert!(few.tips.iter().all(|t| t.w >= MIN_TIP_W)); + assert_eq!(few.tips_suppressed, 0); + } + /// A sample placed deeper than the walk still has to land somewhere, or the chart quietly /// undercounts. #[test] diff --git a/rust/crates/du-web/src/routes/tree.rs b/rust/crates/du-web/src/routes/tree.rs index 478cd1e2..6f4f184b 100644 --- a/rust/crates/du-web/src/routes/tree.rs +++ b/rust/crates/du-web/src/routes/tree.rs @@ -527,14 +527,24 @@ async fn clade_geo_data( /// a continent and says nothing about where a *line* went. Nodes older than this render as a /// signpost down to their eligible children rather than as a chart. const ORIGINS_MAX_YBP: i32 = 1500; -/// Safety bound on the subtree walk — not a display choice. The drawn depth is decided by -/// `origins_layout::prune_to_origins`, which keeps only the branches carrying an origin. -const ORIGINS_DEPTH: i32 = 40; +/// Safety bound on the subtree *walk* — not a display choice. Composition rolls up from every +/// branch inside it, however deep. +const ORIGINS_WALK: i32 = 40; +/// Levels of branching actually drawn. This is a legibility bound, not a data one: R-DF85 has 266 +/// branches over 12 levels and drew an 11,220px canvas — six screens of horizontal scrolling — +/// because canvas width is driven by the number of leaf branches. Folding below this depth keeps +/// every man in his ancestor's composition while collapsing the shape to one screen. +const ORIGINS_DEPTH_DEFAULT: i32 = 4; +const ORIGINS_DEPTH_MIN: i32 = 1; +const ORIGINS_DEPTH_MAX: i32 = 8; +const ORIGINS_DEPTH_OPTIONS: [i32; 6] = [2, 3, 4, 5, 6, 8]; #[derive(Deserialize)] struct OriginsQuery { /// `country` (default) · `admin` · `place`. level: Option<String>, + /// Branching levels drawn; clamped to [`ORIGINS_DEPTH_MIN`]..=[`ORIGINS_DEPTH_MAX`]. + depth: Option<i32>, } fn origins_level(s: Option<&str>) -> place::Level { @@ -581,13 +591,17 @@ async fn origins( .await? .ok_or_else(|| AppError::NotFound(format!("no haplogroup {name}")))?; let level = origins_level(q.level.as_deref()); + let depth = q + .depth + .unwrap_or(ORIGINS_DEPTH_DEFAULT) + .clamp(ORIGINS_DEPTH_MIN, ORIGINS_DEPTH_MAX); let crumbs = build_crumbs(&st.pool, dna_type, base_path, &name).await?; // The era gate. A clade older than the ceiling gets its eligible children rather than a chart // whose every band would read "Europe" — the reader is sent down, not turned away. let too_old = node.tmrca_ybp.is_none_or(|t| t > ORIGINS_MAX_YBP); if too_old { - let window = du_db::haplogroup::subtree_window(&st.pool, dna_type, &name, ORIGINS_DEPTH).await?; + let window = du_db::haplogroup::subtree_window(&st.pool, dna_type, &name, ORIGINS_WALK).await?; let eligible: Vec<Crumb> = window .iter() .filter(|n| n.id != node.id.0 && n.tmrca_ybp.is_some_and(|t| t <= ORIGINS_MAX_YBP)) @@ -604,6 +618,8 @@ async fn origins( base_path, name, level: level_code(level), + depth, + depth_options: ORIGINS_DEPTH_OPTIONS.iter().map(|&d| (d, d == depth)).collect(), tmrca_ybp: node.tmrca_ybp, max_ybp: ORIGINS_MAX_YBP, crumbs, @@ -617,10 +633,17 @@ async fn origins( // The whole subtree, not a display window: `origins_layout` prunes to the branches that // actually carry an origin, so the drawn depth follows the data instead of a fixed number — // and no sample is lost for sitting below an arbitrary cut-off. - let window = du_db::haplogroup::subtree_window(&st.pool, dna_type, &name, ORIGINS_DEPTH).await?; - // De-novo auto-named nodes must not surface publicly here any more than on the tree itself. - // They stay in the input marked `hidden`, so the ancestor walk is unbroken and their men are - // attributed to the nearest named branch rather than dropped. + let window = du_db::haplogroup::subtree_window(&st.pool, dna_type, &name, ORIGINS_WALK).await?; + // Two reasons a branch is marked `hidden`, both meaning "not drawn, but still walked": + // + // * de-novo auto-named nodes, which must not surface publicly here any more than on the + // tree itself; + // * anything past the display depth. + // + // Either way it stays in the input, so the ancestor walk is unbroken and its men are + // attributed to the nearest drawn branch instead of being lost. That is what makes the depth + // bound a *legibility* bound rather than a data one: the composition is identical at every + // depth, only the visible branching changes. let nodes: Vec<origins_layout::Node> = window .iter() .map(|n| origins_layout::Node { @@ -629,7 +652,7 @@ async fn origins( parent_id: n.parent_id, formed_ybp: n.formed_ybp, tmrca_ybp: n.tmrca_ybp, - hidden: is_private_node(&n.name) || is_uuid_label(&n.name), + hidden: n.depth > depth || is_private_node(&n.name) || is_uuid_label(&n.name), }) .collect(); @@ -644,6 +667,8 @@ async fn origins( base_path, name, level: level_code(level), + depth, + depth_options: ORIGINS_DEPTH_OPTIONS.iter().map(|&d| (d, d == depth)).collect(), tmrca_ybp: node.tmrca_ybp, max_ybp: ORIGINS_MAX_YBP, crumbs, @@ -674,6 +699,10 @@ struct OriginsPageTemplate { base_path: &'static str, name: String, level: &'static str, + /// Branching levels drawn — a legibility bound; composition is unaffected by it. + depth: i32, + /// (depth value, is-current) for the selector. + depth_options: Vec<(i32, bool)>, tmrca_ybp: Option<i32>, max_ybp: i32, crumbs: Vec<Crumb>, diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html index 7684fb03..5e702abb 100644 --- a/rust/crates/du-web/templates/tree/origins.html +++ b/rust/crates/du-web/templates/tree/origins.html @@ -20,11 +20,24 @@ <h1 class="h3 mb-0 me-auto">{{ t.get("tree.origins.title") }} · {{ name }}</h1> not a repaint of survivors. #} <div class="btn-group btn-group-sm" role="group" aria-label="{{ t.get("tree.origins.level") }}"> <a class="btn btn-outline-secondary{% if level == "country" %} active{% endif %}" - href="{{ base_path }}/node/{{ name }}/origins?level=country">{{ t.get("tree.origins.level.country") }}</a> + href="{{ base_path }}/node/{{ name }}/origins?level=country&depth={{ depth }}">{{ t.get("tree.origins.level.country") }}</a> <a class="btn btn-outline-secondary{% if level == "admin" %} active{% endif %}" - href="{{ base_path }}/node/{{ name }}/origins?level=admin">{{ t.get("tree.origins.level.admin") }}</a> + href="{{ base_path }}/node/{{ name }}/origins?level=admin&depth={{ depth }}">{{ t.get("tree.origins.level.admin") }}</a> <a class="btn btn-outline-secondary{% if level == "place" %} active{% endif %}" - href="{{ base_path }}/node/{{ name }}/origins?level=place">{{ t.get("tree.origins.level.place") }}</a> + href="{{ base_path }}/node/{{ name }}/origins?level=place&depth={{ depth }}">{{ t.get("tree.origins.level.place") }}</a> + </div> + + {# Branching levels drawn. A legibility control only — the composition of every band is the + same at every depth, because folded branches are counted into their nearest drawn ancestor. #} + <div class="d-flex align-items-center gap-1"> + <label for="origins-depth" class="form-label small text-muted mb-0">{{ t.get("tree.depth") }}</label> + <select id="origins-depth" class="form-select form-select-sm" style="width:auto" + aria-label="{{ t.get("tree.depth") }}" + onchange="location.search='?level={{ level }}&depth='+this.value"> + {% for opt in depth_options %} + <option value="{{ opt.0 }}"{% if opt.1 %} selected{% endif %}>{{ opt.0 }}</option> + {% endfor %} + </select> </div> <a class="btn btn-sm btn-outline-secondary" href="{{ base_path }}?root={{ name }}">{{ t.get("tree.origins.back") }}</a> @@ -55,6 +68,9 @@ <h1 class="h3 mb-0 me-auto">{{ t.get("tree.origins.title") }} · {{ name }}</h1> {% if l.pruned > 0 %} · {{ l.pruned }} {{ t.get("tree.origins.pruned") }} {% endif %} + {% if l.tips_suppressed > 0 %} + · {{ l.tips_suppressed }} {{ t.get("tree.origins.tipshidden") }} + {% endif %} </p> {% if l.legend.len() >= 2 %} @@ -63,7 +79,7 @@ <h1 class="h3 mb-0 me-auto">{{ t.get("tree.origins.title") }} · {{ name }}</h1> {% for e in l.legend %} <span class="origins-legend-item"> <span class="origins-swatch s{{ e.slot }}"></span> - {% if let Some(lbl) = e.label %}{{ lbl }}{% else %}{{ t.get("tree.origins.unknown") }}{% endif %} + {% if let Some(lbl) = e.label %}{{ lbl }}{% else if e.unknown %}{{ t.get("tree.origins.unknown") }}{% else %}{{ t.get("tree.origins.other") }}{% endif %} <span class="text-muted">{{ e.count }}</span> </span> {% endfor %} @@ -103,16 +119,22 @@ <h1 class="h3 mb-0 me-auto">{{ t.get("tree.origins.title") }} · {{ name }}</h1> {% for s in b.segments %} <rect x="{{ s.x }}" y="{{ b.y }}" width="{{ s.w }}" height="{{ b.h }}" class="origins-seg s{{ s.slot }}" rx="2"> - <title>{% if let Some(lbl) = s.label %}{{ lbl }}{% else %}{{ t.get("tree.origins.other") }}{% endif %} · {{ s.count }} + {% if let Some(lbl) = s.label %}{{ lbl }}{% else if s.unknown %}{{ t.get("tree.origins.unknown") }}{% else %}{{ t.get("tree.origins.other") }}{% endif %} · {{ s.count }} {% endfor %} - {{ b.name }} — {{ b.with_origin }} {{ t.get("tree.origins.with") }}, {{ b.without_origin }} {{ t.get("tree.origins.without") }} + {{ b.name }} — {{ b.with_origin }} {{ t.get("tree.origins.with") }}, {{ b.without_origin }} {{ t.get("tree.origins.without") }}{% if b.has_more %} · {{ t.get("tree.origins.more") }}{% endif %} {# Direct label: the relief the light-mode contrast warning obliges. Text wears text tokens, never the series colour. #} {% if !b.cramped %} {{ b.label }} {% endif %} + {# Branches were folded into this band. Marked so "simple" is never confused with + "not shown" — click through to draw them. #} + {% if b.has_more %} + + + {% endif %} {% endfor %} @@ -142,7 +164,7 @@

{{ t.get("tree.origins.title") }} · {{ name }}

{% for e in l.legend %} - {% if let Some(lbl) = e.label %}{{ lbl }}{% else %}{{ t.get("tree.origins.unknown") }}{% endif %} + {% if let Some(lbl) = e.label %}{{ lbl }}{% else if e.unknown %}{{ t.get("tree.origins.unknown") }}{% else %}{{ t.get("tree.origins.other") }}{% endif %} {{ e.count }} {% endfor %} diff --git a/rust/locales/en.txt b/rust/locales/en.txt index 7f5db16a..62140f7c 100644 --- a/rust/locales/en.txt +++ b/rust/locales/en.txt @@ -180,6 +180,8 @@ tree.origins.tmrca=Clade TMRCA tree.origins.placed=placed samples tree.origins.unresolved=with no published origin tree.origins.pruned=branches hidden (no published origin below them) +tree.origins.tipshidden=men shown in the composition only (too narrow to label) +tree.origins.more=has further branches below — click to open tree.origins.with=with an origin tree.origins.without=without one tree.origins.unknown=No locality recorded diff --git a/rust/locales/es.txt b/rust/locales/es.txt index d240b1d2..75ce5b98 100644 --- a/rust/locales/es.txt +++ b/rust/locales/es.txt @@ -132,6 +132,8 @@ tree.origins.tmrca=TMRCA del clado tree.origins.placed=muestras situadas tree.origins.unresolved=sin origen publicado tree.origins.pruned=ramas ocultas (sin origen publicado debajo) +tree.origins.tipshidden=hombres mostrados solo en la composición (demasiado estrecho para etiquetar) +tree.origins.more=tiene más ramas debajo — pulse para abrir tree.origins.with=con origen tree.origins.without=sin origen tree.origins.unknown=Sin localidad registrada diff --git a/rust/locales/fr.txt b/rust/locales/fr.txt index 551f792a..0b59b7d7 100644 --- a/rust/locales/fr.txt +++ b/rust/locales/fr.txt @@ -132,6 +132,8 @@ tree.origins.tmrca=TMRCA du clade tree.origins.placed=échantillons placés tree.origins.unresolved=sans origine publiée tree.origins.pruned=branches masquées (aucune origine publiée en dessous) +tree.origins.tipshidden=hommes indiqués seulement dans la composition (trop étroit pour étiqueter) +tree.origins.more=comporte d’autres branches en dessous — cliquez pour ouvrir tree.origins.with=avec une origine tree.origins.without=sans origine tree.origins.unknown=Aucune localité enregistrée From 34ee964ac934e8c0a1f8160f518011a5f5b473c6 Mon Sep 17 00:00:00 2001 From: James Kane Date: Thu, 6 Aug 2026 15:03:24 -0500 Subject: [PATCH 06/10] feat(tree): blocks show their equivalent SNPs; origin colour belongs to the men MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections to what a block means. A BLOCK IS ITS SNPs. The branch's phylogenetically equivalent mutations are unordered — nothing separates them — so the list *is* the block, exactly as the Big Tree draws it. They were missing entirely; a block was a bare rectangle. `variant_names_for` fetches them for the whole window in one query rather than one per branch, and they flow into as many columns as the block's width allows and as many rows as its height allows. The block is never grown to fit the list: its height is elapsed time and has to stay on the shared axis, so what does not fit is reported as "+N" on a line of its own. BLOCKS ARE NO LONGER TINTED BY ORIGIN. Colouring a clade by the composition of its descendants asserted something the data does not support — a branch has no locality, only the men standing on it do, and a modal-origin tint reads as a claim about the whole lineage. The colour now lives exactly where the claim does: on each man's box, keyed to his own most distant known ancestor. The legend and table still carry the composition, which is what explains those colours, so `Segment` became a tally rather than a drawn mark and lost its geometry. Two layout bugs that only rendering showed. A leaf block is exactly LEAF_W wide, which is narrower than one preferred SNP column, so flooring the column count gave it zero columns and dropped its SNPs — the common case, not an edge one; a block now always gets at least one column, sized to what it actually has. And the SNP list was anchored to the block's padding rather than to the name's baseline, so every block opened with its name and first SNP overprinted. Co-Authored-By: Claude Opus 5 (1M context) --- rust/crates/du-db/src/haplogroup.rs | 31 +++ rust/crates/du-web/assets/main.css | 4 + rust/crates/du-web/src/origins_layout.rs | 242 +++++++++++++----- rust/crates/du-web/src/routes/tree.rs | 5 + .../crates/du-web/templates/tree/origins.html | 17 +- rust/locales/en.txt | 1 + rust/locales/es.txt | 1 + rust/locales/fr.txt | 1 + 8 files changed, 231 insertions(+), 71 deletions(-) diff --git a/rust/crates/du-db/src/haplogroup.rs b/rust/crates/du-db/src/haplogroup.rs index bbb160fe..e428c5a9 100644 --- a/rust/crates/du-db/src/haplogroup.rs +++ b/rust/crates/du-db/src/haplogroup.rs @@ -1209,3 +1209,34 @@ pub async fn pathway(pool: &PgPool, called_name: &str, dna_type: DnaType) -> Res .collect(); Ok(Pathway { dna_type, called_name: called_name.to_string(), resolved_name: Some(resolved), steps }) } + +/// Defining-SNP names for a set of nodes, in one query — the block contents for the origins +/// icicle, which draws a whole subtree at once and must not issue a query per branch. +/// +/// Unnamed variants (`canonical_name IS NULL` — folded legacy homoplasy/duplicate rows) are +/// excluded, matching [`merge_candidates`]: they name nothing a reader could look up. +/// Names come back sorted so a block's contents are stable between requests. +pub async fn variant_names_for( + pool: &PgPool, + ids: &[i64], +) -> Result>, DbError> { + use std::collections::HashMap; + if ids.is_empty() { + return Ok(HashMap::new()); + } + let rows: Vec<(i64, String)> = sqlx::query_as( + "SELECT hv.haplogroup_id, v.canonical_name FROM tree.haplogroup_variant hv \ + JOIN core.variant v ON v.id = hv.variant_id \ + WHERE hv.valid_until IS NULL AND hv.haplogroup_id = ANY($1) \ + AND v.canonical_name IS NOT NULL \ + ORDER BY hv.haplogroup_id, v.canonical_name", + ) + .bind(ids) + .fetch_all(pool) + .await?; + let mut out: HashMap> = HashMap::new(); + for (id, name) in rows { + out.entry(id).or_default().push(name); + } + Ok(out) +} diff --git a/rust/crates/du-web/assets/main.css b/rust/crates/du-web/assets/main.css index ffb19469..c76a4c45 100644 --- a/rust/crates/du-web/assets/main.css +++ b/rust/crates/du-web/assets/main.css @@ -257,3 +257,7 @@ code { color: #495057; background-color: #f8f9fa; } .origins-swatch.s7 { background: var(--o-7); } .origins-swatch.s8 { background: var(--o-8); } .origins-more { fill: var(--bs-secondary-color, #6c757d); font-weight: 700; } +/* Block contents: the branch's equivalent SNPs. Recessive against the block, and never coloured + by anything — a mutation has no locality. */ +.origins-snp { fill: var(--bs-body-color, #212529); opacity: .75; font-family: var(--bs-font-monospace, monospace); } +.origins-snp-more { fill: var(--bs-secondary-color, #6c757d); font-style: italic; } diff --git a/rust/crates/du-web/src/origins_layout.rs b/rust/crates/du-web/src/origins_layout.rs index d63db8a1..69271a50 100644 --- a/rust/crates/du-web/src/origins_layout.rs +++ b/rust/crates/du-web/src/origins_layout.rs @@ -3,9 +3,15 @@ //! //! The shape is ytree.net's / the Big Tree's: depth runs **down** the page, each branch is a band //! spanning the horizontal extent of its descendants, and children sit flush beneath their parent -//! so *containment* carries descent and no connector is drawn. What differs is the fill — instead -//! of the branch's SNPs, a band is a **stacked composition of where its men's ancestors came -//! from**. +//! so *containment* carries descent and no connector is drawn. A block shows **its equivalent +//! SNPs**, exactly as the Big Tree does: the mutations on that branch are unordered, so the list +//! *is* the block. +//! +//! **Origin is carried by the men, not by the branches.** An early cut tinted each block by the +//! composition of its descendants' origins; it is gone. A branch has no locality of its own — only +//! the men standing on it do — and tinting a whole clade by the modal origin of its subtree +//! asserted something the data does not support. The colour now lives exactly where the claim +//! does: on each man's box, keyed to his own most distant known ancestor. //! //! **Time is absolute, not cumulative.** A band's top and bottom are dates on one linear axis, so //! its height *is* its duration; nothing accumulates and nothing drifts. @@ -51,17 +57,29 @@ const MIN_TIP_W: f64 = 26.0; const TIP_GAP: f64 = 10.0; const GUTTER_W: f64 = 54.0; const MARGIN: f64 = 8.0; -/// Gap between stacked segments, per the mark spec — segments are separated by surface, not by a -/// stroke. -const SEG_GAP: f64 = 2.0; +/// One line of SNP text inside a block. +const SNP_LINE_H: f64 = 11.0; +/// Column width for the SNP list. Names run `A9185` to `14405732-C-T`; this holds the common ones +/// and lets the fitter ellipsize the rest. +const SNP_COL_W: f64 = 72.0; +/// Padding inside a block before its SNP list starts. +const SNP_PAD: f64 = 4.0; +/// Baseline of the branch-name line inside a block, matching the template's `dy`. The SNP list +/// starts a full line below it — anchoring it to `SNP_PAD` instead put the first SNP 4px from the +/// name's baseline, so every block opened with its name and first SNP overprinted. +const NAME_BASELINE: f64 = 11.0; /// Categorical slots available before folding into "Other". The palette is fixed-order and never /// cycled; a ninth locality is not given a generated hue. pub const MAX_SERIES: usize = 8; -/// One locality's share of a band. `slot` indexes the fixed categorical palette (1..=[`MAX_SERIES`]); -/// `0` is the reserved neutral used for both "Other" and "no locality recorded", which are -/// absences rather than identities and must not wear a categorical hue. +/// One locality's share of a clade. `slot` indexes the fixed categorical palette +/// (1..=[`MAX_SERIES`]); `0` is the reserved neutral used for both "Other" and "no locality +/// recorded", which are absences rather than identities and must not wear a categorical hue. +/// +/// This is a tally, not a drawn mark: blocks are no longer tinted by composition, so a segment +/// carries no geometry. It feeds the legend and the table, which are what explain the men's +/// colours. #[derive(Debug, Clone, PartialEq)] pub struct Segment { pub label: Option, @@ -71,8 +89,6 @@ pub struct Segment { /// `false` with no label = "Other" (localities past the palette). They share a colour but not /// a meaning, so the legend and tooltips must not call both the same thing. pub unknown: bool, - pub x: f64, - pub w: f64, } /// One laid-out branch. @@ -92,7 +108,13 @@ pub struct Band { pub with_origin: usize, /// Placed samples at or below it that do not — always drawn, never omitted. pub without_origin: usize, - pub segments: Vec, + /// The branch's SNP names, placed inside the block. Flowed into columns, and cut to what the + /// block's height and width can hold — the height means elapsed time, so it is not stretched + /// to fit a long list. + pub snps: Vec, + /// Total equivalent SNPs on the branch, and how many the block had room for. When they differ + /// the block says so rather than quietly showing a subset. + pub snp_total: usize, /// True when the band is too short to letter — the view puts its label in the tooltip only. pub cramped: bool, /// `name` fitted to the band's width. The full name is always in the band's ``. @@ -103,6 +125,14 @@ pub struct Band { pub has_more: bool, } +/// One SNP name placed inside a block. +#[derive(Debug, Clone, PartialEq)] +pub struct SnpCell { + pub name: String, + pub x: f64, + pub y: f64, +} + /// One man, as a leaf below the branch he is placed on. #[derive(Debug, Clone, PartialEq)] pub struct Tip { @@ -166,6 +196,9 @@ pub struct Node { /// A de-novo auto-named node, which must not surface publicly. It stays in the input so the /// ancestor walk is unbroken, and its men are attributed to the nearest named ancestor. pub hidden: bool, + /// The branch's phylogenetically equivalent SNPs. Order is not information — they cannot be + /// separated — so they are listed alphabetically for a stable render. + pub snps: Vec<String>, } /// Attribute every origin to the nearest **visible** branch at or above where its sample is @@ -378,20 +411,18 @@ fn segments_for( label: Some(l.clone()), count: n, unknown: false, - x: 0.0, - w: 0.0, }) .collect(); let with_origin: usize = segs.iter().map(|s| s.count).sum::<usize>() + other; if other > 0 { - segs.push(Segment { label: None, count: other, slot: 0, unknown: false, x: 0.0, w: 0.0 }); + segs.push(Segment { label: None, count: other, slot: 0, unknown: false }); } // "No locality recorded" is DRAWN, always last, so absence sits at the same end of every bar // and bands can be compared by eye. Leaving it as bare background — which is what happened // until a real clade was rendered — made the chart disagree with its own legend, and made a // branch whose men are unrecorded look like a branch with fewer men. if unknown > 0 { - segs.push(Segment { label: None, count: unknown, slot: 0, unknown: true, x: 0.0, w: 0.0 }); + segs.push(Segment { label: None, count: unknown, slot: 0, unknown: true }); } (segs, with_origin, unknown) } @@ -469,22 +500,13 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl // reads as unmeasured rather than brief. _ => (fallback_top[i], UNDATED_H), }; - let (mut segs, with_origin, without_origin) = + let (_, with_origin, without_origin) = segments_for(comp.get(&n.id).unwrap_or(&HashMap::new()), &slots); - // Widths proportional to the composition, with a surface gap between segments. - let inner = extent[i]; - let total = with_origin + without_origin; - if total > 0 { - let gaps = SEG_GAP * segs.len().saturating_sub(1) as f64; - let usable = (inner - gaps).max(0.0); - let mut x = left[i]; - for s in &mut segs { - s.w = usable * (s.count as f64 / total as f64); - s.x = x; - x += s.w + SEG_GAP; - } - } + // The block's SNPs, flowed into as many columns as its width allows and as many rows as + // its height allows. Height is elapsed time, so the block is never stretched to fit the + // list; what does not fit is reported instead (`snp_total`). + let snps = flow_snps(&n.snps, left[i], y, extent[i], h); bands.push(Band { id: n.id, label: fit(&n.name, extent[i], 10.0), @@ -498,7 +520,8 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl tmrca_ybp: n.tmrca_ybp, with_origin, without_origin, - segments: segs, + snps, + snp_total: n.snps.len(), cramped: h < 14.0, has_more: has_more.contains(&n.id), }); @@ -572,6 +595,43 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl } } +/// Place a block's equivalent SNPs inside it, filling columns top-to-bottom then left-to-right. +/// +/// The list is cut to what the block can hold rather than the block being grown to hold the list: +/// a block's height is elapsed time and must stay on the shared axis. The caller reports the total +/// so a truncated list is never mistaken for a complete one. +fn flow_snps(names: &[String], x: f64, y: f64, w: f64, h: f64) -> Vec<SnpCell> { + // The first line is the branch name; SNPs start a full line below its baseline. + let top = y + NAME_BASELINE + SNP_LINE_H; + let mut rows = (((y + h - SNP_PAD) - top) / SNP_LINE_H).floor().max(0.0) as usize; + let inner = w - 2.0 * SNP_PAD; + if rows == 0 || inner <= 0.0 { + return Vec::new(); + } + let cols_avail = (inner / SNP_COL_W).floor().max(1.0) as usize; + // Give the "+N did not fit" marker its own line rather than letting it overprint the last + // SNP — the marker is the thing that keeps a cut list from reading as a complete one. + if names.len() > rows * cols_avail && rows > 1 { + rows -= 1; + } + // At least one column whenever the block has any width at all. A leaf block is exactly + // `LEAF_W` wide, which is narrower than the preferred column, so a plain floor gave it zero + // columns and dropped its SNPs entirely — the common case, not an edge case. The column then + // takes the width actually available and `fit` ellipsizes into it. + let cols = cols_avail; + let col_w = inner / cols as f64; + names + .iter() + .take(rows * cols) + .enumerate() + .map(|(k, name)| SnpCell { + name: fit(name, col_w, 9.0), + x: x + SNP_PAD + (k / rows) as f64 * col_w, + y: top + (k % rows) as f64 * SNP_LINE_H, + }) + .collect() +} + /// Approximate width of one character of the SVG label font, as a fraction of its size. The /// canvas has no text metrics, so labels are fitted arithmetically; erring narrow would clip text /// that fits, erring wide lets it spill. @@ -658,7 +718,15 @@ mod tests { use uuid::Uuid; fn node(id: i64, name: &str, parent: Option<i64>, formed: Option<i32>, tmrca: Option<i32>) -> Node { - Node { id, name: name.into(), parent_id: parent, formed_ybp: formed, tmrca_ybp: tmrca, hidden: false } + Node { + id, + name: name.into(), + parent_id: parent, + formed_ybp: formed, + tmrca_ybp: tmrca, + hidden: false, + snps: Vec::new(), + } } fn hidden(id: i64, name: &str, parent: Option<i64>, formed: Option<i32>, tmrca: Option<i32>) -> Node { @@ -717,20 +785,16 @@ mod tests { let laid = layout(&tree(), &origins, Level::Admin, 2); let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); assert_eq!(root.with_origin, 1); - assert_eq!(root.without_origin, 1, "counted"); + assert_eq!(root.without_origin, 1, "counted, not dropped"); - // And DRAWN — left as bare background it made the chart disagree with its own legend, and - // a branch of unrecorded men looked like a branch with fewer men. - let absent = root.segments.iter().find(|s| s.unknown).expect("an unknown segment exists"); + // And named in the legend, which is where the men's colours are explained. An absence is + // its own entry — a branch of unrecorded men must not read as a branch with fewer men. + let absent = laid.legend.iter().find(|e| e.unknown).expect("an unknown legend entry"); assert_eq!(absent.count, 1); assert_eq!(absent.slot, 0, "an absence never wears a categorical hue"); - assert!(absent.w > 0.0, "it occupies real width"); - // Absence sits last in every bar, so bands can be compared by eye. - assert!(root.segments.last().unwrap().unknown); - // Segments now account for the whole band. - let covered: f64 = root.segments.iter().map(|s| s.w).sum::<f64>() - + SEG_GAP * (root.segments.len() - 1) as f64; - assert!((covered - root.w).abs() < 0.01, "the bar is fully accounted for"); + assert!(laid.legend.last().unwrap().unknown, "absence sorts last"); + // The man himself is drawn in the neutral slot. + assert!(laid.tips.iter().any(|t| t.slot == 0)); } /// Slot 0 carries two different things. They share a colour but not a meaning, and the legend @@ -864,9 +928,11 @@ mod tests { assert!(a.x + a.w <= bb.x + 0.01, "siblings are disjoint"); } - /// Segment widths are proportional and stay inside the band, gaps included. + /// A branch has no locality of its own — only the men standing on it do. Blocks were once + /// tinted by the modal origin of their subtree, which asserted something the data does not + /// support; the colour now lives only on the men. #[test] - fn segments_are_proportional_and_stay_within_the_band() { + fn blocks_are_never_coloured_by_origin_only_the_men_are() { let origins = vec![ origin(2, "Cork, Co. Cork, Ireland"), origin(2, "Cork, Co. Cork, Ireland"), @@ -874,13 +940,64 @@ mod tests { bare(3), ]; let laid = layout(&tree(), &origins, Level::Admin, 4); - let root = laid.bands.iter().find(|b| b.id == 1).unwrap(); - let cork = root.segments.iter().find(|s| s.label.as_deref() == Some("Co. Cork")).unwrap(); - let kerry = root.segments.iter().find(|s| s.label.as_deref() == Some("Co. Kerry")).unwrap(); - assert!((cork.w / kerry.w - 2.0).abs() < 0.01, "2 Cork to 1 Kerry"); - for s in &root.segments { - assert!(s.x >= root.x - 0.01 && s.x + s.w <= root.x + root.w + 0.01); + // Every man carries a slot; Cork and Kerry are different colours, the unrecorded man is 0. + let slots: Vec<usize> = laid.tips.iter().map(|t| t.slot).collect(); + assert_eq!(slots.len(), 4); + assert!(slots.contains(&0), "the unrecorded man wears the neutral"); + assert!(slots.iter().filter(|&&s| s == 1).count() == 2, "both Cork men share a slot"); + // And the legend still explains those colours, with the counts. + let cork = laid.legend.iter().find(|e| e.label.as_deref() == Some("Co. Cork")).unwrap(); + assert_eq!(cork.count, 2); + assert_eq!(laid.legend.iter().find(|e| e.label.as_deref() == Some("Co. Kerry")).unwrap().count, 1); + } + + /// A block shows its equivalent SNPs — the mutations are unordered, so the list IS the block. + /// It is cut to what the block holds rather than the block being grown, because the height is + /// elapsed time and has to stay on the shared axis. + #[test] + fn a_block_lists_its_equivalent_snps_and_reports_what_did_not_fit() { + let mut nodes = tree(); + nodes[1].snps = (0..80).map(|i| format!("FGC{i:05}")).collect(); + nodes[2].snps = vec!["A9185".into(), "BY23498".into()]; + let laid = layout(&nodes, &[origin(2, "Ireland"), origin(3, "Ireland")], Level::Country, 2); + + let short = laid.bands.iter().find(|b| b.id == 3).unwrap(); + assert_eq!(short.snp_total, 2); + assert_eq!(short.snps.len(), 2, "a short list fits whole"); + assert!(short.snps.iter().any(|c| c.name == "A9185")); + + let long = laid.bands.iter().find(|b| b.id == 2).unwrap(); + assert_eq!(long.snp_total, 80); + assert!(long.snps.len() < 80, "80 SNPs cannot fit a 555-year block"); + assert!(!long.snps.is_empty()); + // Every placed name stays inside its block. + for c in &long.snps { + assert!(c.x >= long.x - 0.01 && c.x <= long.x + long.w + 0.01); + assert!(c.y >= long.y - 0.01 && c.y <= long.y + long.h + 0.01); } + // Columns fill top-to-bottom, then left-to-right. + if long.snps.len() > 1 { + assert!(long.snps[1].y > long.snps[0].y || long.snps[1].x > long.snps[0].x); + } + // The list clears the branch-name line. Anchored to the padding instead, every block + // opened with its name and its first SNP overprinted. + assert!(long.snps[0].y >= long.y + NAME_BASELINE + SNP_LINE_H - 0.01); + // And the "+N" marker has a line of its own at the foot of the block. + assert!(long.snps.iter().all(|c| c.y <= long.y + long.h - SNP_LINE_H)); + } + + /// A leaf block is exactly `LEAF_W` wide — narrower than one preferred column. Flooring the + /// column count gave it zero columns and silently dropped its SNPs, which is the common case + /// rather than an edge one. + #[test] + fn a_leaf_width_block_still_gets_one_column() { + let mut nodes = tree(); + nodes[1].snps = vec!["A9185".into(), "BY23498".into(), "FT225347".into()]; + let laid = layout(&nodes, &[origin(2, "Ireland")], Level::Country, 1); + let leaf = laid.bands.iter().find(|b| b.id == 2).unwrap(); + assert_eq!(leaf.w, LEAF_W, "the narrowest a block gets"); + assert!(!leaf.snps.is_empty(), "its SNPs are drawn, not dropped"); + assert!(leaf.snps.iter().all(|c| c.x >= leaf.x && c.x <= leaf.x + leaf.w)); } /// The reader is told what the chart could not account for. @@ -989,19 +1106,18 @@ mod tests { let root_of = |l: &Laid| l.bands.iter().find(|b| b.id == 1).unwrap().clone(); assert_eq!(root_of(&full).with_origin, root_of(&folded).with_origin, "3 men either way"); - let seg = |l: &Laid, id: i64, name: &str| { - l.bands - .iter() - .find(|b| b.id == id) - .unwrap() - .segments - .iter() - .find(|s| s.label.as_deref() == Some(name)) - .map(|s| s.count) + // The legend is the root's composition, so it is the thing folding must not change. + let count = |l: &Laid, name: &str| { + l.legend.iter().find(|e| e.label.as_deref() == Some(name)).map(|e| e.count) }; - assert_eq!(seg(&full, 1, "Co. Cork"), Some(2)); - assert_eq!(seg(&folded, 1, "Co. Cork"), Some(2), "unchanged by folding"); - assert_eq!(seg(&folded, 1, "Co. Kerry"), Some(1)); + assert_eq!(count(&full, "Co. Cork"), Some(2)); + assert_eq!(count(&folded, "Co. Cork"), Some(2), "unchanged by folding"); + assert_eq!(count(&folded, "Co. Kerry"), Some(1)); + // Every man is still accounted for, wherever his branch got folded to. Folding puts all + // three onto one leaf block, where they no longer each fit a legible box — so they move + // from `tips` to `tips_suppressed` rather than disappearing. + assert_eq!(full.tips.len() + full.tips_suppressed, 3); + assert_eq!(folded.tips.len() + folded.tips_suppressed, 3); // R-Deep is gone from the drawing, and its men are now R-Mid's. assert!(folded.bands.iter().all(|b| b.id != 3)); assert_eq!(folded.bands.iter().find(|b| b.id == 2).unwrap().with_origin, 3); diff --git a/rust/crates/du-web/src/routes/tree.rs b/rust/crates/du-web/src/routes/tree.rs index 6f4f184b..0dbed2b8 100644 --- a/rust/crates/du-web/src/routes/tree.rs +++ b/rust/crates/du-web/src/routes/tree.rs @@ -644,6 +644,10 @@ async fn origins( // attributed to the nearest drawn branch instead of being lost. That is what makes the depth // bound a *legibility* bound rather than a data one: the composition is identical at every // depth, only the visible branching changes. + // The blocks' contents: each branch's phylogenetically equivalent SNPs, in one query rather + // than one per branch. + let node_ids: Vec<i64> = window.iter().map(|n| n.id).collect(); + let mut snps_of = du_db::haplogroup::variant_names_for(&st.pool, &node_ids).await?; let nodes: Vec<origins_layout::Node> = window .iter() .map(|n| origins_layout::Node { @@ -653,6 +657,7 @@ async fn origins( formed_ybp: n.formed_ybp, tmrca_ybp: n.tmrca_ybp, hidden: n.depth > depth || is_private_node(&n.name) || is_uuid_label(&n.name), + snps: snps_of.remove(&n.id).unwrap_or_default(), }) .collect(); diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html index 5e702abb..9b25d190 100644 --- a/rust/crates/du-web/templates/tree/origins.html +++ b/rust/crates/du-web/templates/tree/origins.html @@ -115,14 +115,7 @@ <h1 class="h3 mb-0 me-auto">{{ t.get("tree.origins.title") }} · {{ name }}</h1> <rect x="{{ b.x }}" y="{{ b.y }}" width="{{ b.w }}" height="{{ b.h }}" fill="url(#undated)" rx="2"></rect> {% endif %} - {# Stacked composition. Segments are separated by surface, not by a stroke. #} - {% for s in b.segments %} - <rect x="{{ s.x }}" y="{{ b.y }}" width="{{ s.w }}" height="{{ b.h }}" - class="origins-seg s{{ s.slot }}" rx="2"> - <title>{% if let Some(lbl) = s.label %}{{ lbl }}{% else if s.unknown %}{{ t.get("tree.origins.unknown") }}{% else %}{{ t.get("tree.origins.other") }}{% endif %} · {{ s.count }} - - {% endfor %} - {{ b.name }} — {{ b.with_origin }} {{ t.get("tree.origins.with") }}, {{ b.without_origin }} {{ t.get("tree.origins.without") }}{% if b.has_more %} · {{ t.get("tree.origins.more") }}{% endif %} + {{ b.name }} — {{ b.snp_total }} {{ t.get("tree.origins.snps") }} · {{ b.with_origin }} {{ t.get("tree.origins.with") }}, {{ b.without_origin }} {{ t.get("tree.origins.without") }}{% if b.has_more %} · {{ t.get("tree.origins.more") }}{% endif %} {# Direct label: the relief the light-mode contrast warning obliges. Text wears text tokens, never the series colour. #} @@ -131,6 +124,14 @@

{{ t.get("tree.origins.title") }} · {{ name }}

{% endif %} {# Branches were folded into this band. Marked so "simple" is never confused with "not shown" — click through to draw them. #} + {# The branch's equivalent SNPs — the mutations are unordered, so the list IS the block. #} + {% for sn in b.snps %} + {{ sn.name }} + {% endfor %} + {% if b.snp_total > b.snps.len() %} + +{{ b.snp_total - b.snps.len() }} + {% endif %} {% if b.has_more %} + diff --git a/rust/locales/en.txt b/rust/locales/en.txt index 62140f7c..f8040b3d 100644 --- a/rust/locales/en.txt +++ b/rust/locales/en.txt @@ -182,6 +182,7 @@ tree.origins.unresolved=with no published origin tree.origins.pruned=branches hidden (no published origin below them) tree.origins.tipshidden=men shown in the composition only (too narrow to label) tree.origins.more=has further branches below — click to open +tree.origins.snps=equivalent SNPs tree.origins.with=with an origin tree.origins.without=without one tree.origins.unknown=No locality recorded diff --git a/rust/locales/es.txt b/rust/locales/es.txt index 75ce5b98..38ad0fd2 100644 --- a/rust/locales/es.txt +++ b/rust/locales/es.txt @@ -134,6 +134,7 @@ tree.origins.unresolved=sin origen publicado tree.origins.pruned=ramas ocultas (sin origen publicado debajo) tree.origins.tipshidden=hombres mostrados solo en la composición (demasiado estrecho para etiquetar) tree.origins.more=tiene más ramas debajo — pulse para abrir +tree.origins.snps=SNP equivalentes tree.origins.with=con origen tree.origins.without=sin origen tree.origins.unknown=Sin localidad registrada diff --git a/rust/locales/fr.txt b/rust/locales/fr.txt index 0b59b7d7..78fb2b62 100644 --- a/rust/locales/fr.txt +++ b/rust/locales/fr.txt @@ -134,6 +134,7 @@ tree.origins.unresolved=sans origine publiée tree.origins.pruned=branches masquées (aucune origine publiée en dessous) tree.origins.tipshidden=hommes indiqués seulement dans la composition (trop étroit pour étiqueter) tree.origins.more=comporte d’autres branches en dessous — cliquez pour ouvrir +tree.origins.snps=SNP équivalents tree.origins.with=avec une origine tree.origins.without=sans origine tree.origins.unknown=Aucune localité enregistrée From 79305f970d055a0b8bf8dd1f8704ccd58cfb983b Mon Sep 17 00:00:00 2001 From: James Kane Date: Thu, 6 Aug 2026 15:48:26 -0500 Subject: [PATCH 07/10] fix(tree): size a block by its SNP count, not by the age model MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A block with 16 equivalent SNPs was drawing as an 18px sliver. Sizing blocks by elapsed years does not survive contact with the data, and both obvious forms of it fail: * a node's own formed_ybp → its own tmrca_ybp draws children ON TOP of their parents — the two are independent point estimates under no monotonicity constraint, agreeing on 898 of 10,252 edges while 4,243 (41%) have the child forming before its parent's split (fixed earlier, in 4ffc3b8); * parent TMRCA → own TMRCA is monotone, so containment held — but it is degenerate. `formed_ybp == tmrca_ybp` on 41% of terminal branches and 26.5% of internal ones, collapsing those branches to a point. On R-DF85 at depth 4 that left 30 of 75 blocks unable to show a single one of their SNPs: R-BY18328 got 3px of span for 9 mutations, R-BY170664 16 SNPs in an 18px sliver, and a quarter of all blocks sat pinned at the minimum height. So the block is sized to its SNPs — one line each, nothing elided — which is what BACKLOG's own block-tree convention already says and what the Big Tree does. The list is the block; truncating it shortens the box, and a shortened box misreports how long the branch ran unbroken. Vertical position becomes cumulative, and the gutter rules off in mutations rather than calendar years. This loses nothing as a time axis. Mutations accrue at a roughly steady rate, and measured on this very tree branch length tracks SNP count at r = 0.975, about 69 years per mutation — better behaved than the per-branch estimate it replaces, which is missing or degenerate exactly where a block most needs a height. Ages keep their two real jobs: gating the view to the genealogical era, and labelling each block. Result on R-DF85 at depth 4: every one of 468 SNPs drawn, 0 blocks hiding any, and the canvas got SHORTER (706px → 439px) because blocks now take the height their content needs instead of whatever a sparse time axis handed them. The "+N did not fit" marker is gone with the problem it reported. Co-Authored-By: Claude Opus 5 (1M context) --- .../proposals/ancestral-origin-icicle.md | 55 ++-- rust/crates/du-web/assets/main.css | 1 + rust/crates/du-web/src/origins_layout.rs | 303 +++++++++--------- .../crates/du-web/templates/tree/origins.html | 12 +- rust/locales/en.txt | 1 + rust/locales/es.txt | 1 + rust/locales/fr.txt | 1 + 7 files changed, 201 insertions(+), 173 deletions(-) diff --git a/documents/proposals/ancestral-origin-icicle.md b/documents/proposals/ancestral-origin-icicle.md index 10621fa7..6de62885 100644 --- a/documents/proposals/ancestral-origin-icicle.md +++ b/documents/proposals/ancestral-origin-icicle.md @@ -113,25 +113,42 @@ library, matching `tree_layout.rs`. - **Geometry**: depth → y; each node a rect spanning its subtree's horizontal extent; children flush against the parent's underside, so containment carries descent and no connector is drawn. -- **Height = elapsed years**: a branch spans **its parent's TMRCA → its own TMRCA**, on one absolute - calendar axis. This is the deliberate divergence from Navigator's SNP-count height, and the reason - to build the view here: the AppView has ages (`tmrca_ybp` on 10,257 of 11,422 Y nodes) and the - framing is temporal. Nodes with no age draw at a minimum height, hatched, and are excluded from - the ruler — visible, not silently normal. - - **Do not use a node's own `formed_ybp` for the top of its band.** It is the obvious choice and it - is wrong: `formed_ybp` and the parent's `tmrca_ybp` are independent point estimates under no - monotonicity constraint, and on the live tree they agree on only **898 of 10,252 edges** while - **4,243 (41%) have the child forming earlier than its parent's split**. Driving geometry from it - draws children on top of their parents — caught by rendering the real tree, where `R-A13318` - (formed 1622) landed at exactly its parent `R-S764`'s y. Parent-TMRCA → own-TMRCA has **zero** - inversions over the same edges, so containment holds by construction. -- **Fill = stacked locality composition** of the placed samples at or below the block, at the - selected level (Country / Admin1 / Place), with **"no locality recorded" always its own visible - slice**. A view of who published is not a view of where a branch is from, and the difference must - be on screen. -- **Tips**: one leaf box per placed sample carrying an origin — `Kane · Co. Clare`, coloured to - match. Never the kit id. +- **A block shows its equivalent SNPs, and its height is their count** — one line each, nothing + elided, exactly as the Big Tree draws it. The mutations on a branch are unordered, so the list + *is* the block. Vertical position is therefore cumulative: how far down a block sits is the + mutations accrued along the path to it, and the left gutter rules that off in SNPs. + + **Do not size blocks by the age model.** Both obvious forms were tried against real data and both + fail: + + 1. *A node's own `formed_ybp` → its own `tmrca_ybp`.* These are independent point estimates under + no monotonicity constraint; they agree with the parent's TMRCA on only **898 of 10,252 edges**, + and **4,243 (41%)** have the child forming *earlier* than its parent's split — so children + draw on top of their parents. `R-A13318` (formed 1622) landed at exactly its parent + `R-S764`'s y. + 2. *Parent TMRCA → own TMRCA.* Monotone, so containment holds — but degenerate: + `formed_ybp == tmrca_ybp` on **41% of terminal branches and 26.5% of internal ones**, collapsing + the branch to a point. On R-DF85 at depth 4 that left **30 of 75 blocks unable to show a single + one of their SNPs** — `R-BY18328` got 3px of span for 9 mutations, `R-BY170664` 16 SNPs in an + 18px sliver. + + SNP count never degenerates, and it is still a time axis: measured on this tree, branch length + tracks SNP count at **r = 0.975, ≈69 years per mutation**. Ages keep their two real jobs — gating + the view to the genealogical era, and labelling each block — they simply do not drive geometry, + because a per-branch estimate is precisely what is missing or degenerate when a block most needs + a height. + +- **Colour belongs to the men, not the branches.** An early cut tinted each block by the composition + of its descendants' origins. That asserts something the data does not support: a branch has no + locality, only the men standing on it do, and a modal-origin tint reads as a claim about the whole + lineage. Colour lives on each man's box, keyed to his own MDKA; the legend and table carry the + composition that explains those colours. +- **Tips**: one leaf box per placed sample carrying an origin — `Kane · Co. Clare`, coloured by his + own locality. Never the kit id. A man too narrow to label is counted rather than drawn as an + unreadable sliver, and the count is stated. +- **"No locality recorded" is a visible category**, in the legend and the table, never bare + background. A view of who published is not a view of where a branch is from, and the difference + must be on screen. - **Colours**: categorical, colourblind-safe, legible in both themes; assigned by frequency rank *within the rendered subtree* (deterministic, tie-broken on name), top N distinct + a neutral "other". diff --git a/rust/crates/du-web/assets/main.css b/rust/crates/du-web/assets/main.css index c76a4c45..a230cea0 100644 --- a/rust/crates/du-web/assets/main.css +++ b/rust/crates/du-web/assets/main.css @@ -261,3 +261,4 @@ code { color: #495057; background-color: #f8f9fa; } by anything — a mutation has no locality. */ .origins-snp { fill: var(--bs-body-color, #212529); opacity: .75; font-family: var(--bs-font-monospace, monospace); } .origins-snp-more { fill: var(--bs-secondary-color, #6c757d); font-style: italic; } +.origins-tick-unit { fill: var(--bs-secondary-color, #6c757d); font-style: italic; } diff --git a/rust/crates/du-web/src/origins_layout.rs b/rust/crates/du-web/src/origins_layout.rs index 69271a50..1d3a7814 100644 --- a/rust/crates/du-web/src/origins_layout.rs +++ b/rust/crates/du-web/src/origins_layout.rs @@ -13,22 +13,20 @@ //! asserted something the data does not support. The colour now lives exactly where the claim //! does: on each man's box, keyed to his own most distant known ancestor. //! -//! **Time is absolute, not cumulative.** A band's top and bottom are dates on one linear axis, so -//! its height *is* its duration; nothing accumulates and nothing drifts. +//! **A block's height is its SNP count, and nothing is elided.** Every equivalent SNP gets a line, +//! so the box's height *is* how long that branch ran unbroken, and vertical position is cumulative: +//! how far down a block sits is the mutations accrued along the path to it. //! -//! A branch spans **its parent's TMRCA → its own TMRCA**: from the split that brought it into -//! existence as a separate line, to the point it began diversifying itself. That specific pairing -//! is deliberate. The obvious choice — a node's own `formed_ybp` → its own `tmrca_ybp` — draws -//! children *above* their parents on real data: `formed_ybp` and the parent's `tmrca_ybp` are -//! independent point estimates under no monotonicity constraint, and on the live Y tree they are -//! equal on only 898 of 10,252 edges while **4,243 (41%) have the child forming earlier than its -//! parent's split**. Parent-TMRCA → own-TMRCA has zero inversions across the same 10,252 edges, so -//! containment is guaranteed by construction rather than by hope. `formed_ybp` is still reported, -//! on the band itself, where a reader can see the estimate without the geometry depending on it. +//! This is not a stylistic choice — sizing blocks by the age model was tried and does not survive +//! contact with the data. `formed_ybp == tmrca_ybp` on **41% of terminal branches and 26.5% of +//! internal ones**, collapsing those branches to a point; on R-DF85 at depth 4 that left **30 of +//! 75 blocks unable to show a single one of their SNPs**, `R-BY18328` getting 3px of span for 9 +//! mutations. SNP count never degenerates. And it is still a time axis: measured on this tree, +//! branch length tracks SNP count at **r = 0.975, about 69 years per mutation**. //! -//! Undated nodes (16% of sample-bearing nodes) are **not** silently normalized: they hang from -//! their parent at a minimum height and are hatched, so an unmeasured branch never reads as a -//! short one. +//! Ages are not discarded — they gate the view to the genealogical era and label each block — they +//! just do not drive geometry, because a per-branch estimate is exactly the thing that is missing +//! or degenerate when a block most needs a height. //! //! Pure: no DB, no `Ui`, no template. Every function here is testable without a canvas. @@ -39,15 +37,8 @@ use std::collections::HashMap; /// Canvas geometry at scale 1. const LEAF_W: f64 = 74.0; const H_GAP: f64 = 4.0; -/// Pixels per year of elapsed time. The genealogical era is ~1,500 years, so this puts a full -/// gated subtree in roughly 900px. -const PX_PER_YEAR: f64 = 0.6; -/// A band never collapses below this, however brief the branch — a 20-year branch must still be -/// clickable and still show its composition. -const MIN_BAND_H: f64 = 18.0; -/// Height given to a band with no age at all. Deliberately equal to the minimum so it cannot be -/// mistaken for a *measured* short branch; the hatch is what distinguishes it. -const UNDATED_H: f64 = MIN_BAND_H; +/// Ruler graduation interval, in mutations. +const TICK_SNPS: usize = 5; /// Sample tips hang in a band below the youngest branch. const TIP_H: f64 = 16.0; /// Narrowest a man's box may be and still carry a readable label. Below it the box is dropped and @@ -59,15 +50,10 @@ const GUTTER_W: f64 = 54.0; const MARGIN: f64 = 8.0; /// One line of SNP text inside a block. const SNP_LINE_H: f64 = 11.0; -/// Column width for the SNP list. Names run `A9185` to `14405732-C-T`; this holds the common ones -/// and lets the fitter ellipsize the rest. -const SNP_COL_W: f64 = 72.0; /// Padding inside a block before its SNP list starts. const SNP_PAD: f64 = 4.0; -/// Baseline of the branch-name line inside a block, matching the template's `dy`. The SNP list -/// starts a full line below it — anchoring it to `SNP_PAD` instead put the first SNP 4px from the -/// name's baseline, so every block opened with its name and first SNP overprinted. -const NAME_BASELINE: f64 = 11.0; +/// The branch-name line at the top of every block. +const NAME_LINE_H: f64 = 12.0; /// Categorical slots available before folding into "Other". The palette is fixed-order and never /// cycled; a ninth locality is not given a generated hue. @@ -146,13 +132,11 @@ pub struct Tip { pub h: f64, } -/// A ruler graduation on the absolute time axis. +/// A ruler graduation: mutations accumulated along the lineage to this point. #[derive(Debug, Clone, PartialEq)] pub struct Tick { pub y: f64, - pub ybp: i32, - /// Calendar-era label (`"1500 CE"`), because the genealogical era reads in calendar years. - pub label: String, + pub snps: usize, } /// A legend row. Present whenever the chart carries two or more series — identity is never @@ -467,46 +451,25 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl extent[i] = extent[i].max(kids + gaps); } - // The time axis. The root's formation is the top of the canvas; the present is the bottom, so - // tips land on "now" and every band sits at its true date. - let top_ybp = root.formed_ybp.or(root.tmrca_ybp).unwrap_or(0); - let y_of = |ybp: i32| MARGIN + (top_ybp - ybp).max(0) as f64 * PX_PER_YEAR; - // A branch begins at its parent's TMRCA — the split that created it. See the module header for - // why this is not the node's own `formed_ybp`. - let tmrca_of: HashMap = nodes.iter().filter_map(|n| Some((n.id, n.tmrca_ybp?))).collect(); - let split_of: HashMap = nodes - .iter() - .filter_map(|n| Some((n.id, *tmrca_of.get(&n.parent_id?)?))) - .collect(); - - // Pass 2 (pre-order): x from the parent's band, y from the node's own dates. + // Pass 2 (pre-order): x from the parent's band, y stacked directly beneath it. let mut bands = Vec::with_capacity(nodes.len()); let mut tips = Vec::new(); let mut left = vec![0.0f64; nodes.len()]; - let mut fallback_top = vec![0.0f64; nodes.len()]; + let mut top = vec![0.0f64; nodes.len()]; left[root_i] = GUTTER_W; - fallback_top[root_i] = MARGIN; + top[root_i] = MARGIN; let mut stack = vec![root_i]; let mut deepest = 0.0f64; while let Some(i) = stack.pop() { let n = &nodes[i]; - // Top: the parent's TMRCA for a child, the node's own formation for the root. Bottom: this - // node's TMRCA. The pairing is monotone, so `h` can never come out negative. - let top_ybp_of = split_of.get(&n.id).copied().or(n.formed_ybp); + let (y, h) = (top[i], block_height(n.snps.len())); let dated = n.tmrca_ybp.is_some(); - let (y, h) = match (top_ybp_of, n.tmrca_ybp) { - (Some(from), Some(to)) => (y_of(from), (y_of(to) - y_of(from)).max(MIN_BAND_H)), - // Undated: hang from wherever the parent ended, at the minimum height. Hatched, so it - // reads as unmeasured rather than brief. - _ => (fallback_top[i], UNDATED_H), - }; let (_, with_origin, without_origin) = segments_for(comp.get(&n.id).unwrap_or(&HashMap::new()), &slots); - // The block's SNPs, flowed into as many columns as its width allows and as many rows as - // its height allows. Height is elapsed time, so the block is never stretched to fit the - // list; what does not fit is reported instead (`snp_total`). - let snps = flow_snps(&n.snps, left[i], y, extent[i], h); + // One SNP per line, nothing elided — the block is sized to its list, so the list always + // fits and `snp_total` can never exceed what is drawn. + let snps = flow_snps(&n.snps, left[i], y, extent[i]); bands.push(Band { id: n.id, label: fit(&n.name, extent[i], 10.0), @@ -530,7 +493,9 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl let mut cx = left[i]; for &c in &children[i] { left[c] = cx; - fallback_top[c] = y + h; + // Children sit flush beneath their parent: containment carries descent, and vertical + // position is cumulative mutations along the lineage. + top[c] = y + h; cx += extent[c] + H_GAP; stack.push(c); } @@ -577,7 +542,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl } } - let ticks = ruler(top_ybp, deepest, &y_of); + let ticks = ruler(nodes, &bands); let legend = legend_for(&root_comp, &slots); let width = GUTTER_W + extent[root_i] + MARGIN * 2.0; let height = tip_y + TIP_H + MARGIN; @@ -595,39 +560,32 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl } } -/// Place a block's equivalent SNPs inside it, filling columns top-to-bottom then left-to-right. +/// Height a block needs: its name line plus one line per equivalent SNP. /// -/// The list is cut to what the block can hold rather than the block being grown to hold the list: -/// a block's height is elapsed time and must stay on the shared axis. The caller reports the total -/// so a truncated list is never mistaken for a complete one. -fn flow_snps(names: &[String], x: f64, y: f64, w: f64, h: f64) -> Vec { - // The first line is the branch name; SNPs start a full line below its baseline. - let top = y + NAME_BASELINE + SNP_LINE_H; - let mut rows = (((y + h - SNP_PAD) - top) / SNP_LINE_H).floor().max(0.0) as usize; - let inner = w - 2.0 * SNP_PAD; - if rows == 0 || inner <= 0.0 { - return Vec::new(); - } - let cols_avail = (inner / SNP_COL_W).floor().max(1.0) as usize; - // Give the "+N did not fit" marker its own line rather than letting it overprint the last - // SNP — the marker is the thing that keeps a cut list from reading as a complete one. - if names.len() > rows * cols_avail && rows > 1 { - rows -= 1; - } - // At least one column whenever the block has any width at all. A leaf block is exactly - // `LEAF_W` wide, which is narrower than the preferred column, so a plain floor gave it zero - // columns and dropped its SNPs entirely — the common case, not an edge case. The column then - // takes the width actually available and `fit` ellipsizes into it. - let cols = cols_avail; - let col_w = inner / cols as f64; +/// **The block is sized to its SNPs, not to a clock.** An earlier cut sized it by elapsed years +/// (parent TMRCA → own TMRCA) and that fails on real data: `formed_ybp == tmrca_ybp` on **41% of +/// terminal branches and 26.5% of internal ones**, collapsing the branch to a point. On R-DF85 at +/// depth 4 that left **30 of 75 blocks unable to show a single one of their SNPs** — `R-BY18328` +/// got 3px of span for 9 mutations. SNP count never degenerates, and because mutations accrue at a +/// roughly steady rate it still reads as elapsed time: measured on this very tree, branch length +/// correlates with SNP count at **r = 0.975**, about **69 years per SNP**. +pub fn block_height(snps: usize) -> f64 { + 2.0 * SNP_PAD + NAME_LINE_H + snps as f64 * SNP_LINE_H +} + +/// Place a block's equivalent SNPs: one per line, in order, nothing elided. +/// +/// Truncating would shorten the box, and a shortened box misreports how long the branch ran +/// unbroken — the same reason the block is sized to the list rather than the list cut to the box. +fn flow_snps(names: &[String], x: f64, y: f64, w: f64) -> Vec { + let top = y + SNP_PAD + NAME_LINE_H; names .iter() - .take(rows * cols) .enumerate() .map(|(k, name)| SnpCell { - name: fit(name, col_w, 9.0), - x: x + SNP_PAD + (k / rows) as f64 * col_w, - y: top + (k % rows) as f64 * SNP_LINE_H, + name: fit(name, w - 2.0 * SNP_PAD, 9.0), + x: x + SNP_PAD, + y: top + (k as f64 + 0.8) * SNP_LINE_H, }) .collect() } @@ -669,38 +627,57 @@ fn post_order(children: &[Vec], root: usize) -> Vec { out } -/// Graduations at a round interval chosen so the axis carries roughly 6–10 of them. -fn ruler(top_ybp: i32, bottom_px: f64, y_of: &dyn Fn(i32) -> f64) -> Vec { - if top_ybp <= 0 { - return Vec::new(); +/// Graduations down the **deepest lineage** — the one that accrued the most mutations, and so the +/// one that reaches furthest down the canvas. +/// +/// Ticks are computed by walking that lineage rather than spaced evenly, because evenly spaced +/// would be wrong: each block spends one line on its name, so a fixed pixels-per-SNP scale drifts +/// by a line per generation. Placing each graduation inside the block that contains it keeps the +/// axis honest — the ticks come out nearly regular, and where they do not, the irregularity is +/// real. +fn ruler(nodes: &[Node], bands: &[Band]) -> Vec { + let index: HashMap = nodes.iter().enumerate().map(|(i, n)| (n.id, i)).collect(); + let band_of: HashMap = bands.iter().map(|b| (b.id, b)).collect(); + + // Cumulative mutations to the bottom of each block, so "deepest" means most mutations rather + // than most generations — one long branch outranks several short ones. + let mut cum = vec![0usize; nodes.len()]; + let mut best = (0usize, 0usize); + for (i, n) in nodes.iter().enumerate() { + let above = n.parent_id.and_then(|p| index.get(&p)).map(|&p| cum[p]).unwrap_or(0); + cum[i] = above + n.snps.len(); + if cum[i] > best.0 { + best = (cum[i], i); + } } - let step = [50, 100, 200, 250, 500, 1000, 2000, 5000] - .into_iter() - .find(|s| top_ybp / s <= 10) - .unwrap_or(10_000); + // Walk back up from the deepest block, then read the chain root-first. + let mut chain = Vec::new(); + let mut at = Some(best.1); + while let Some(i) = at { + chain.push(i); + at = nodes[i].parent_id.and_then(|p| index.get(&p)).copied(); + } + chain.reverse(); + let mut ticks = Vec::new(); - let mut ybp = (top_ybp / step) * step; - while ybp >= 0 { - let y = y_of(ybp); - if y <= bottom_px + 1.0 { - ticks.push(Tick { y, ybp, label: era_label(ybp) }); + let mut seen = 0usize; + for i in chain { + let n = &nodes[i]; + let Some(b) = band_of.get(&n.id) else { continue }; + let body_top = b.y + SNP_PAD + NAME_LINE_H; + let mut k = (seen / TICK_SNPS + 1) * TICK_SNPS; + while k <= seen + n.snps.len() { + ticks.push(Tick { + y: body_top + (k - seen) as f64 * SNP_LINE_H, + snps: k, + }); + k += TICK_SNPS; } - ybp -= step; + seen += n.snps.len(); } ticks } -/// `ybp` → a calendar-era label. The genealogical era is read in calendar years, not in years -/// before present, and 1950 is the radiocarbon reference the rest of the tree uses. -fn era_label(ybp: i32) -> String { - let year = 1950 - ybp; - if year > 0 { - format!("{year} CE") - } else { - format!("{} BCE", 1 - year) - } -} - fn legend_for(root_comp: &HashMap, usize>, slots: &HashMap) -> Vec { // The legend is exactly the root band's segments — including the drawn "Other" and // "no locality recorded" bars, which is what keeps chart and legend from disagreeing. @@ -847,19 +824,44 @@ mod tests { /// Height is duration on one absolute axis: a branch that ran twice as long is twice as tall, /// wherever it sits in the tree. + /// A block's height IS its SNP count — one line per equivalent mutation, nothing elided — and + /// vertical position is cumulative, so how far down a block sits is the mutations accrued + /// along the path to it. #[test] - fn band_height_is_elapsed_time_on_an_absolute_axis() { - let laid = layout(&tree(), &[origin(2, "Ireland"), origin(3, "Scotland")], Level::Country, 2); + fn block_height_is_its_snp_count_and_position_is_cumulative() { + let mut nodes = tree(); + nodes[0].snps = (0..3).map(|i| format!("S{i}")).collect(); + nodes[1].snps = (0..9).map(|i| format!("A{i}")).collect(); + nodes[2].snps = (0..2).map(|i| format!("B{i}")).collect(); + let laid = layout(&nodes, &[origin(2, "Ireland"), origin(3, "Scotland")], Level::Country, 2); let b = |id: i64| laid.bands.iter().find(|b| b.id == id).unwrap().clone(); - // Both children begin at the parent's split (1355) and run to their own TMRCA. - assert!((b(2).h - 555.0 * PX_PER_YEAR).abs() < 0.01, "1355→800"); - assert!((b(3).h - 755.0 * PX_PER_YEAR).abs() < 0.01, "1355→600"); - // Siblings share that split, so their tops align exactly. + + assert_eq!(b(2).h, block_height(9)); + assert_eq!(b(3).h, block_height(2)); + assert!(b(2).h > b(3).h, "nine mutations is a longer branch than two"); + // Siblings share their parent's bottom, so they start level. assert!((b(2).y - b(3).y).abs() < 0.01); - // And each child starts where the parent's diversification did. + // And each child hangs flush beneath the parent — containment, no connector. assert!((b(2).y - (b(1).y + b(1).h)).abs() < 0.01); } + /// The bug that prompted the change. `formed_ybp == tmrca_ybp` on 41% of terminal branches, so + /// an age-sized block collapsed to nothing while still carrying a full SNP list — 30 of 75 + /// blocks on R-DF85 could not show a single mutation. Height must not depend on that estimate. + #[test] + fn a_branch_whose_age_estimate_collapsed_still_shows_every_snp() { + // R-BY18328's real numbers: formed == tmrca, and its parent is 3 years older. + let nodes = vec![ + node(1, "R-FT191128", None, Some(906), Some(906)), + Node { snps: (0..9).map(|i| format!("BY{i}")).collect(), ..node(2, "R-BY18328", Some(1), Some(903), Some(903)) }, + ]; + let laid = layout(&nodes, &[origin(2, "Ireland")], Level::Country, 1); + let b = laid.bands.iter().find(|b| b.id == 2).unwrap(); + assert_eq!(b.snps.len(), 9, "all nine drawn"); + assert_eq!(b.snp_total, 9); + assert_eq!(b.h, block_height(9)); + } + /// The bug that rendering the real tree exposed. `formed_ybp` and the parent's `tmrca_ybp` are /// independent estimates under no monotonicity constraint: on the live Y tree 4,243 of 10,252 /// edges have a child forming *earlier* than its parent's split. Driving the geometry from @@ -903,16 +905,18 @@ mod tests { } } - /// An unmeasured branch must not read as a short one. + /// A branch with no age estimate is still a branch with mutations. Geometry no longer depends + /// on the age at all, so it draws at full height like any other; `dated` survives only to + /// label it. #[test] - fn an_undated_branch_is_hatched_at_the_minimum_height() { + fn an_undated_branch_still_gets_its_full_height() { let mut nodes = tree(); - nodes.push(node(4, "R-C", Some(2), None, None)); + nodes.push(Node { snps: (0..16).map(|i| format!("BY{i}")).collect(), ..node(4, "R-C", Some(2), None, None) }); let laid = layout(&nodes, &[origin(4, "Ireland")], Level::Country, 1); let c = laid.bands.iter().find(|b| b.id == 4).unwrap(); - assert!(!c.dated); - assert_eq!(c.h, UNDATED_H); - // It hangs off its parent rather than floating at the top of the canvas. + assert!(!c.dated, "still flagged as unmeasured"); + assert_eq!(c.h, block_height(16)); + assert_eq!(c.snps.len(), 16, "16 SNPs in an 18px sliver was the bug"); let parent = laid.bands.iter().find(|b| b.id == 2).unwrap(); assert!((c.y - (parent.y + parent.h)).abs() < 0.01); } @@ -968,29 +972,23 @@ mod tests { let long = laid.bands.iter().find(|b| b.id == 2).unwrap(); assert_eq!(long.snp_total, 80); - assert!(long.snps.len() < 80, "80 SNPs cannot fit a 555-year block"); - assert!(!long.snps.is_empty()); + assert_eq!(long.snps.len(), 80, "nothing is elided — the block grows to its list"); // Every placed name stays inside its block. for c in &long.snps { assert!(c.x >= long.x - 0.01 && c.x <= long.x + long.w + 0.01); assert!(c.y >= long.y - 0.01 && c.y <= long.y + long.h + 0.01); } - // Columns fill top-to-bottom, then left-to-right. - if long.snps.len() > 1 { - assert!(long.snps[1].y > long.snps[0].y || long.snps[1].x > long.snps[0].x); - } + // One per line, in order, and every one inside its block. + assert!(long.snps.windows(2).all(|w| w[1].y > w[0].y)); + assert!(long.snps.iter().all(|c| c.y > long.y && c.y <= long.y + long.h)); // The list clears the branch-name line. Anchored to the padding instead, every block // opened with its name and its first SNP overprinted. - assert!(long.snps[0].y >= long.y + NAME_BASELINE + SNP_LINE_H - 0.01); - // And the "+N" marker has a line of its own at the foot of the block. - assert!(long.snps.iter().all(|c| c.y <= long.y + long.h - SNP_LINE_H)); + assert!(long.snps[0].y >= long.y + SNP_PAD + NAME_LINE_H); } - /// A leaf block is exactly `LEAF_W` wide — narrower than one preferred column. Flooring the - /// column count gave it zero columns and silently dropped its SNPs, which is the common case - /// rather than an edge one. + /// A leaf block is exactly `LEAF_W` wide, so its SNP names must be fitted to that, not dropped. #[test] - fn a_leaf_width_block_still_gets_one_column() { + fn a_leaf_width_block_still_shows_its_snps() { let mut nodes = tree(); nodes[1].snps = vec!["A9185".into(), "BY23498".into(), "FT225347".into()]; let laid = layout(&nodes, &[origin(2, "Ireland")], Level::Country, 1); @@ -1007,14 +1005,23 @@ mod tests { assert_eq!(laid.unresolved, 16, "17 placed, 1 with a published origin"); } + /// The ruler counts mutations down the deepest lineage. It is walked rather than spaced + /// evenly, because each block spends a line on its name and a fixed scale would drift. #[test] - fn the_ruler_reads_in_calendar_years() { - let laid = layout(&tree(), &[], Level::Country, 0); + fn the_ruler_counts_mutations_down_the_deepest_lineage() { + let mut nodes = tree(); + nodes[0].snps = (0..4).map(|i| format!("S{i}")).collect(); + nodes[1].snps = (0..12).map(|i| format!("A{i}")).collect(); + nodes[2].snps = (0..2).map(|i| format!("B{i}")).collect(); + let laid = layout(&nodes, &[origin(2, "Ireland"), origin(3, "Ireland")], Level::Country, 2); + assert!(!laid.ticks.is_empty()); assert!(laid.ticks.windows(2).all(|w| w[0].y < w[1].y), "monotone down the page"); - assert_eq!(era_label(1600), "350 CE"); - assert_eq!(era_label(0), "1950 CE"); - assert_eq!(era_label(2000), "51 BCE"); + assert!(laid.ticks.windows(2).all(|w| w[1].snps > w[0].snps)); + // Graduations land on multiples of the interval, and stop at the deepest lineage's total + // (4 + 12 = 16, not 4 + 2). + assert!(laid.ticks.iter().all(|t| t.snps % TICK_SNPS == 0)); + assert_eq!(laid.ticks.last().unwrap().snps, 15); } /// A legend is always available for two or more series — identity is never colour alone. diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html index 9b25d190..ba0ce83d 100644 --- a/rust/crates/du-web/templates/tree/origins.html +++ b/rust/crates/du-web/templates/tree/origins.html @@ -98,13 +98,17 @@

{{ t.get("tree.origins.title") }} · {{ name }}

- {# Time ruler: calendar years, one linear scale for the whole canvas. #} + {# Mutation ruler, walked down the deepest lineage. Mutations accrue at a roughly steady rate + — measured on this tree, branch length tracks SNP count at r=0.975, ~69 years each — so the + axis reads as elapsed time without depending on a per-branch age estimate. #} {% for tk in l.ticks %} - {{ tk.label }} + {{ tk.snps }} {% endfor %} + {{ t.get("tree.origins.axis") }} {% for b in l.bands %} @@ -128,10 +132,6 @@

{{ t.get("tree.origins.title") }} · {{ name }}

{% for sn in b.snps %} {{ sn.name }} {% endfor %} - {% if b.snp_total > b.snps.len() %} - +{{ b.snp_total - b.snps.len() }} - {% endif %} {% if b.has_more %} + diff --git a/rust/locales/en.txt b/rust/locales/en.txt index f8040b3d..35a0072a 100644 --- a/rust/locales/en.txt +++ b/rust/locales/en.txt @@ -183,6 +183,7 @@ tree.origins.pruned=branches hidden (no published origin below them) tree.origins.tipshidden=men shown in the composition only (too narrow to label) tree.origins.more=has further branches below — click to open tree.origins.snps=equivalent SNPs +tree.origins.axis=SNPs tree.origins.with=with an origin tree.origins.without=without one tree.origins.unknown=No locality recorded diff --git a/rust/locales/es.txt b/rust/locales/es.txt index 38ad0fd2..25ba7d09 100644 --- a/rust/locales/es.txt +++ b/rust/locales/es.txt @@ -135,6 +135,7 @@ tree.origins.pruned=ramas ocultas (sin origen publicado debajo) tree.origins.tipshidden=hombres mostrados solo en la composición (demasiado estrecho para etiquetar) tree.origins.more=tiene más ramas debajo — pulse para abrir tree.origins.snps=SNP equivalentes +tree.origins.axis=SNP tree.origins.with=con origen tree.origins.without=sin origen tree.origins.unknown=Sin localidad registrada diff --git a/rust/locales/fr.txt b/rust/locales/fr.txt index 78fb2b62..dc3bd739 100644 --- a/rust/locales/fr.txt +++ b/rust/locales/fr.txt @@ -135,6 +135,7 @@ tree.origins.pruned=branches masquées (aucune origine publiée en dessous) tree.origins.tipshidden=hommes indiqués seulement dans la composition (trop étroit pour étiqueter) tree.origins.more=comporte d’autres branches en dessous — cliquez pour ouvrir tree.origins.snps=SNP équivalents +tree.origins.axis=SNP tree.origins.with=avec une origine tree.origins.without=sans origine tree.origins.unknown=Aucune localité enregistrée From 8c96548db29c994774df42161939d302ff9b74c8 Mon Sep 17 00:00:00 2001 From: James Kane Date: Thu, 6 Aug 2026 17:03:25 -0500 Subject: [PATCH 08/10] fix(tree): drop the cross-hatch, and stop hiding branches that lack origins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two pieces of scaffolding outlived what they were for. THE CROSS-HATCH marked branches with no age estimate. It was justified only because height *was* the age span — an unmeasured branch must not read as a short one. Height is SNP count now (79305f9), so the hatch distinguishes nothing and is just noise on 16% of blocks. Gone, along with the `dated` flag behind it. BRANCHES WITH NO PUBLISHED ORIGIN BENEATH THEM were pruned away. That was how the 11,220px canvas got tamed before the depth bound existed; the depth bound does that job now, and pruning is both redundant and wrong. Origins are an overlay on the tree, not a filter of it — a branch with no locality data is still part of the clade's shape, and dropping it misrepresents the phylogeny to make a sparse overlay look dense. Removing the prune surfaced the tip row's own version of the same confusion. Men were packed into a single row under their branch, so most became slivers narrower than their labels and were dropped and counted instead — 219 of 278 on R-DF85, and with the block tint gone they were then shown nowhere at all. Men now wrap into rows, sized for a readable box first and narrowed only when the branch demands it. But wrapping alone made one depth-4 boundary block absorb 179 folded men and stack them 90 rows deep, a tip row taller than the tree above it. So a man gets a box only where his own branch is drawn: folded upward he still counts in his ancestor's composition, the fold marker is the affordance, and drilling in draws him where he belongs. R-DF85 at depth 4: 76 branches (was 75 — the pruned one is back), all 471 SNPs, no hatch, 64 men drawn and every one legible, canvas 459px. At depth 6 the same view resolves 108 men, which is what drilling in is meant to do. Co-Authored-By: Claude Opus 5 (1M context) --- .../proposals/ancestral-origin-icicle.md | 14 +- rust/crates/du-web/assets/main.css | 4 - rust/crates/du-web/src/origins_layout.rs | 186 ++++++++---------- .../crates/du-web/templates/tree/origins.html | 20 +- rust/locales/en.txt | 2 - rust/locales/es.txt | 2 - rust/locales/fr.txt | 2 - 7 files changed, 93 insertions(+), 137 deletions(-) diff --git a/documents/proposals/ancestral-origin-icicle.md b/documents/proposals/ancestral-origin-icicle.md index 6de62885..48c03d3a 100644 --- a/documents/proposals/ancestral-origin-icicle.md +++ b/documents/proposals/ancestral-origin-icicle.md @@ -155,11 +155,15 @@ library, matching `tree_layout.rs`. - **Era gate**: serves nodes with `tmrca_ybp <= 1500` (adjustable within bounds). Above the cutoff it renders the breadcrumb, one line of explanation, and links down to eligible children rather than drawing a block that means nothing. -- **Pruned to the branches that carry an origin**, with the count reported. A branch with no - published origin beneath it is a column of width and no information: on `R-S764` the unpruned - draw was 175 bands across a 7,944px canvas to show 10 origins; pruned it is 37 bands in 768px. - The drawn depth therefore follows the data rather than a fixed window — which also means no - sample is lost for sitting below a cut-off. +- **Every branch in the window is drawn, origins or not.** Origins are an overlay on the tree, not + a filter of it: a branch with no locality data is still part of the clade's shape, and hiding it + would misrepresent the phylogeny to make a sparse overlay look dense. Legibility is bounded by + the depth selector instead (default 4 levels), which folds rather than drops — folded branches + are marked, and their men still count in their nearest drawn ancestor. +- **A man gets a box only where his own branch is drawn.** Attributed upward from a folded branch + he still counts in the composition, but is not given a box under a branch that is not his; on + R-DF85 one boundary block had otherwise absorbed 179 men and stacked them 90 rows deep. Drilling + in draws him where he belongs. - **De-novo nodes stay hidden but their men still count.** A sample placed on an auto-named node is attributed to the nearest named ancestor, as the public tree already does for sample tips. Dropping it instead made every band above it understate its own composition. diff --git a/rust/crates/du-web/assets/main.css b/rust/crates/du-web/assets/main.css index a230cea0..c6c512a6 100644 --- a/rust/crates/du-web/assets/main.css +++ b/rust/crates/du-web/assets/main.css @@ -230,10 +230,6 @@ code { color: #495057; background-color: #f8f9fa; } .origins-seg.s7 { fill: var(--o-7); } .origins-seg.s8 { fill: var(--o-8); } -/* An unmeasured branch is hatched, so it can never read as a measured short one. */ -.origins-undated-bg { fill: transparent; } -.origins-undated-line { stroke: var(--bs-secondary-color, #6c757d); stroke-width: 1.5; opacity: .5; } - /* Recessive ruler. */ .origins-tick line { stroke: var(--bs-border-color, #dee2e6); stroke-width: 1; } .origins-tick text { fill: var(--bs-secondary-color, #6c757d); } diff --git a/rust/crates/du-web/src/origins_layout.rs b/rust/crates/du-web/src/origins_layout.rs index 1d3a7814..afc68f78 100644 --- a/rust/crates/du-web/src/origins_layout.rs +++ b/rust/crates/du-web/src/origins_layout.rs @@ -41,10 +41,11 @@ const H_GAP: f64 = 4.0; const TICK_SNPS: usize = 5; /// Sample tips hang in a band below the youngest branch. const TIP_H: f64 = 16.0; -/// Narrowest a man's box may be and still carry a readable label. Below it the box is dropped and -/// the man counted instead — a row of 8px slivers hides the composition bar rather than adding to -/// it. +/// Narrowest a man's box may be and still carry a readable label. Men wrap into further rows +/// rather than being packed below it. const MIN_TIP_W: f64 = 26.0; +/// Vertical gap between wrapped rows of men. +const TIP_ROW_GAP: f64 = 2.0; const TIP_GAP: f64 = 10.0; const GUTTER_W: f64 = 54.0; const MARGIN: f64 = 8.0; @@ -86,20 +87,17 @@ pub struct Band { pub y: f64, pub w: f64, pub h: f64, - /// False when the branch has no age estimate — rendered hatched, excluded from the ruler. - pub dated: bool, pub formed_ybp: Option, pub tmrca_ybp: Option, /// Placed samples at or below this branch that carry a published origin. pub with_origin: usize, /// Placed samples at or below it that do not — always drawn, never omitted. pub without_origin: usize, - /// The branch's SNP names, placed inside the block. Flowed into columns, and cut to what the - /// block's height and width can hold — the height means elapsed time, so it is not stretched - /// to fit a long list. + /// The branch's SNP names, one per line inside the block. The block is sized to this list, so + /// it always holds all of them. pub snps: Vec, - /// Total equivalent SNPs on the branch, and how many the block had room for. When they differ - /// the block says so rather than quietly showing a subset. + /// Total equivalent SNPs on the branch — equal to `snps.len()`, kept because the tooltip + /// states the count. pub snp_total: usize, /// True when the band is too short to letter — the view puts its label in the tooltip only. pub cramped: bool, @@ -160,12 +158,6 @@ pub struct Laid { pub legend: Vec, /// Samples under the root with no published origin — the honest denominator. pub unresolved: usize, - /// Branches dropped because no origin sits beneath them. Reported, never silent: a pruned - /// chart that looked complete would misrepresent how much of the clade this is. - pub pruned: usize, - /// Men whose per-man box was too narrow to letter. They remain in their band's composition; - /// only the box is gone. - pub tips_suppressed: usize, } /// The minimum a node needs from the tree window. Mirrors `du_db::haplogroup::WindowNode` so this @@ -185,26 +177,28 @@ pub struct Node { pub snps: Vec, } -/// Attribute every origin to the nearest **visible** branch at or above where its sample is -/// placed, and drop the branches that carry no origin at all. +/// Drop the branches that are not drawn — de-novo auto-named nodes and anything past the display +/// depth — and attribute their men to the nearest branch that *is*. /// -/// Both halves fix silent losses found by rendering the real tree: +/// **Every branch inside the window is kept, origins or not.** An earlier cut also dropped +/// branches with no published origin beneath them; that was how the 11,220px canvas got tamed +/// before the depth bound existed, and it is now both redundant and wrong. Origins are an overlay +/// on the tree, not a filter of it: a branch with no locality data is still part of the clade's +/// shape, and hiding it misrepresents the phylogeny to make a sparse overlay look dense. /// -/// - A sample placed on a hidden (de-novo) node, or below the drawn window, used to contribute to -/// **nothing** — its ancestors never saw it, so every band above it understated its own -/// composition. Climbing to the nearest visible ancestor is what the tree's own private-node -/// collapse does for sample tips, applied to composition. -/// - An origins view is about origins: a branch with none beneath it costs a full column of width -/// and says nothing. On a real clade that was 175 bands and a 7,944px canvas for 10 origins. -/// Pruning is reported, never silent. -/// -/// Retained nodes, origins re-pointed at visible branches, how many branches were pruned, and -/// which retained branches have folded descendants. +/// The remap is what keeps folding honest. A sample on a hidden node, or below the window, used to +/// contribute to **nothing** — its ancestors never saw it, so every block above it understated +/// itself. Climbing to the nearest visible ancestor is what the public tree already does for +/// sample tips, applied to composition. pub struct Pruned { pub nodes: Vec, pub origins: Vec, - pub pruned: usize, + /// Branches that have folded descendants — drawn with a drill-in affordance. pub has_more: std::collections::HashSet, + /// Men whose own branch is drawn. A man attributed upward from a folded branch still counts + /// in his ancestors' composition, but he is not given a box under a branch that is not his: + /// the fold marker is the affordance, and drilling in draws him where he belongs. + pub placed_here: std::collections::HashSet, } pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> Pruned { @@ -213,8 +207,8 @@ pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> Pruned { return Pruned { nodes: Vec::new(), origins: Vec::new(), - pruned: 0, has_more: std::collections::HashSet::new(), + placed_here: std::collections::HashSet::new(), }; }; @@ -236,8 +230,14 @@ pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> Pruned { } root.id }; + let mut placed_here = std::collections::HashSet::new(); let moved: Vec = origins .iter() + .inspect(|o| { + if by_id.get(&o.haplogroup_id).is_some_and(|n| !n.hidden) { + placed_here.insert(o.sample_guid); + } + }) .map(|o| SampleOrigin { haplogroup_id: match by_id.contains_key(&o.haplogroup_id) { true => visible_ancestor(o.haplogroup_id), @@ -248,28 +248,13 @@ pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> Pruned { }) .collect(); - // Keep a branch when an origin sits at or below it. - let mut keep: std::collections::HashSet = std::collections::HashSet::new(); - keep.insert(root.id); - for o in &moved { - let mut at = Some(o.haplogroup_id); - let mut guard = 0; - while let Some(id) = at { - if guard > nodes.len() { - break; - } - keep.insert(id); - at = by_id.get(&id).and_then(|n| n.parent_id); - guard += 1; - } - } - // Re-parent onto the nearest retained ancestor. Dropping a hidden or origin-less branch must - // not orphan the branches beneath it — the chain has to stay walkable or the roll-up and the - // layout both lose everything below the gap. - let retained_id = |id: i64| -> bool { !by_id[&id].hidden && keep.contains(&id) }; + // Re-parent onto the nearest retained ancestor. Dropping a folded branch must not orphan the + // branches beneath it — the chain has to stay walkable or the roll-up and the layout both lose + // everything below the gap. + let retained_id = |id: i64| -> bool { !by_id[&id].hidden }; let retained: Vec = nodes .iter() - .filter(|n| !n.hidden && keep.contains(&n.id)) + .filter(|n| !n.hidden) .map(|n| { let mut p = n.parent_id; let mut guard = 0; @@ -287,11 +272,7 @@ pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> Pruned { Node { parent_id: p, ..n.clone() } }) .collect(); - let pruned = nodes.iter().filter(|n| !n.hidden).count() - retained.len(); - // Which retained branches have folded descendants — every retained ancestor of a hidden node. - // Only *hidden* (depth-folded) nodes count: a branch pruned for carrying no origin adds - // nothing a reader could drill into. let retained_ids: std::collections::HashSet = retained.iter().map(|n| n.id).collect(); let mut has_more = std::collections::HashSet::new(); for n in nodes.iter().filter(|n| n.hidden) { @@ -309,7 +290,7 @@ pub fn prune_to_origins(nodes: &[Node], origins: &[SampleOrigin]) -> Pruned { guard += 1; } } - Pruned { nodes: retained, origins: moved, pruned, has_more } + Pruned { nodes: retained, origins: moved, has_more, placed_here } } /// Roll each sample's locality up to its branch **and every ancestor of that branch**, so a band's @@ -417,7 +398,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl // Attribute origins to visible branches and drop the branches with none beneath them, before // anything is measured — see `prune_to_origins`. let p = prune_to_origins(all_nodes, all_origins); - let (pruned, has_more) = (p.pruned, p.has_more); + let (has_more, placed_here) = (p.has_more, p.placed_here); let (nodes, origins) = (&p.nodes[..], &p.origins[..]); let Some(root) = nodes.iter().find(|n| n.parent_id.is_none()) else { return Laid::default(); @@ -463,7 +444,6 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl while let Some(i) = stack.pop() { let n = &nodes[i]; let (y, h) = (top[i], block_height(n.snps.len())); - let dated = n.tmrca_ybp.is_some(); let (_, with_origin, without_origin) = segments_for(comp.get(&n.id).unwrap_or(&HashMap::new()), &slots); @@ -478,7 +458,6 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl y, w: extent[i], h, - dated, formed_ybp: n.formed_ybp, tmrca_ybp: n.tmrca_ybp, with_origin, @@ -503,25 +482,26 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl // Tips: one per sample with a published origin, under the branch it sits on. let tip_y = deepest + TIP_GAP; + // Only men whose own branch is drawn get a box. One boundary block on R-DF85 had absorbed 179 + // men from everything folded beneath it and stacked them 90 rows deep — a tip row taller than + // the tree it hangs from, under a branch that is not theirs. let mut per_node: HashMap> = HashMap::new(); - for o in origins { + for o in origins.iter().filter(|o| placed_here.contains(&o.sample_guid)) { per_node.entry(o.haplogroup_id).or_default().push(o); } let band_x: HashMap = bands.iter().map(|b| (b.id, (b.x, b.w))).collect(); - let mut tips_suppressed = 0usize; + let mut tip_rows_max = 1usize; for (node_id, mut list) in per_node { let Some(&(bx, bw)) = band_x.get(&node_id) else { continue }; list.sort_by(|a, b| a.sample_guid.cmp(&b.sample_guid)); - let n = list.len() as f64; - let w = (bw - H_GAP * (n - 1.0).max(0.0)) / n; - // Below this a tip is a coloured sliver with no legible label — noise that hides the - // composition bar above it. Those men are still counted in the band; only the per-man box - // is dropped, and the count is reported. - if w < MIN_TIP_W { - tips_suppressed += list.len(); - continue; - } - let w = w.min(LEAF_W); + // Men wrap into rows beneath their branch rather than being squeezed into one. Columns are + // sized for a *readable* box first and only narrowed when the men need more room than the + // branch has: packing to the minimum width instead left every label truncated to a few + // characters even where the branch was wide enough for the whole name. + let roomy = (((bw + H_GAP) / (LEAF_W + H_GAP)).floor() as usize).max(1); + let cols = roomy.min(list.len()).max(1); + let w = ((bw - H_GAP * (cols - 1) as f64) / cols as f64).min(LEAF_W).max(MIN_TIP_W); + tip_rows_max = tip_rows_max.max(list.len().div_ceil(cols)); for (k, o) in list.iter().enumerate() { let locality = o.place.label_at(level); let label = match (&o.surname, locality) { @@ -534,8 +514,8 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl slot: locality.and_then(|l| slots.get(l).copied()).unwrap_or(0), full: label.clone(), label: fit(&label, w, 9.0), - x: bx + k as f64 * (w + H_GAP), - y: tip_y, + x: bx + (k % cols) as f64 * (w + H_GAP), + y: tip_y + (k / cols) as f64 * (TIP_H + TIP_ROW_GAP), w, h: TIP_H, }); @@ -545,7 +525,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl let ticks = ruler(nodes, &bands); let legend = legend_for(&root_comp, &slots); let width = GUTTER_W + extent[root_i] + MARGIN * 2.0; - let height = tip_y + TIP_H + MARGIN; + let height = tip_y + tip_rows_max as f64 * (TIP_H + TIP_ROW_GAP) + MARGIN; let resolved: usize = root_comp.values().sum(); Laid { width, @@ -555,8 +535,6 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl ticks, legend, unresolved: placed_total.saturating_sub(resolved), - pruned, - tips_suppressed, } } @@ -906,15 +884,14 @@ mod tests { } /// A branch with no age estimate is still a branch with mutations. Geometry no longer depends - /// on the age at all, so it draws at full height like any other; `dated` survives only to - /// label it. + /// on the age at all, so it draws like any other — and the cross-hatch that used to mark it is + /// gone with the height model that made the distinction matter. #[test] fn an_undated_branch_still_gets_its_full_height() { let mut nodes = tree(); nodes.push(Node { snps: (0..16).map(|i| format!("BY{i}")).collect(), ..node(4, "R-C", Some(2), None, None) }); let laid = layout(&nodes, &[origin(4, "Ireland")], Level::Country, 1); let c = laid.bands.iter().find(|b| b.id == 4).unwrap(); - assert!(!c.dated, "still flagged as unmeasured"); assert_eq!(c.h, block_height(16)); assert_eq!(c.snps.len(), 16, "16 SNPs in an 18px sliver was the bug"); let parent = laid.bands.iter().find(|b| b.id == 2).unwrap(); @@ -1058,20 +1035,21 @@ mod tests { assert_eq!(layout(&[], &[], Level::Country, 0), Laid::default()); } - /// Found by rendering the real tree: a clade drew 175 bands across 7,944px to show 10 - /// origins. A branch with none beneath it is all width and no information. + /// Origins are an overlay on the tree, not a filter of it. A branch with no locality data + /// beneath it is still part of the clade's shape; dropping it would misrepresent the phylogeny + /// to make a sparse overlay look dense. (An earlier cut did prune them — that was how the + /// 11,220px canvas got tamed before the depth bound existed, and the depth bound does it now.) #[test] - fn branches_with_no_origin_beneath_them_are_pruned_and_counted() { + fn branches_with_no_origin_beneath_them_are_still_drawn() { let mut nodes = tree(); for id in 10..20 { nodes.push(node(id, &format!("R-Empty{id}"), Some(3), Some(600), Some(400))); } let laid = layout(&nodes, &[origin(2, "Ireland")], Level::Country, 1); - // Root + the one branch carrying the origin. R-B and its ten empty children are gone. - assert_eq!(laid.bands.len(), 2); - assert!(laid.bands.iter().all(|b| b.id == 1 || b.id == 2)); - assert_eq!(laid.pruned, 11, "reported, never silent"); - assert!(laid.width < 200.0, "canvas follows the data, not the tree"); + assert_eq!(laid.bands.len(), 13, "every branch in the window is drawn"); + assert!(laid.bands.iter().any(|b| b.id == 15), "including the origin-less ones"); + // The one man still colours only his own tip. + assert_eq!(laid.tips.len(), 1); } /// Also found by rendering: a sample on a de-novo node contributed to *nothing*, so every @@ -1120,11 +1098,13 @@ mod tests { assert_eq!(count(&full, "Co. Cork"), Some(2)); assert_eq!(count(&folded, "Co. Cork"), Some(2), "unchanged by folding"); assert_eq!(count(&folded, "Co. Kerry"), Some(1)); - // Every man is still accounted for, wherever his branch got folded to. Folding puts all - // three onto one leaf block, where they no longer each fit a legible box — so they move - // from `tips` to `tips_suppressed` rather than disappearing. - assert_eq!(full.tips.len() + full.tips_suppressed, 3); - assert_eq!(folded.tips.len() + folded.tips_suppressed, 3); + // Composition is what folding must preserve. Boxes are not: a man whose own branch was + // folded away is counted in his ancestor's tally but not given a box under a branch that + // is not his — the fold marker is the affordance, and drilling in draws him where he + // belongs. Undrawn, one boundary block absorbed 179 men and stacked them 90 rows deep. + assert_eq!(full.tips.len(), 3, "all three branches drawn, all three men drawn"); + assert_eq!(folded.tips.len(), 1, "only the man on the surviving branch"); + assert_eq!(root_of(&folded).with_origin, 3, "but all three still counted"); // R-Deep is gone from the drawing, and its men are now R-Mid's. assert!(folded.bands.iter().all(|b| b.id != 3)); assert_eq!(folded.bands.iter().find(|b| b.id == 2).unwrap().with_origin, 3); @@ -1133,20 +1113,20 @@ mod tests { assert!(!full.bands.iter().find(|b| b.id == 2).unwrap().has_more); } - /// A row of 8px slivers hides the composition bar instead of adding to it. The men stay - /// counted; only the per-man box goes, and the count is reported. + /// Men wrap into rows beneath their branch rather than being squeezed into one. Forcing a + /// single row made each a sliver narrower than its own label, so 219 of 278 on R-DF85 had to + /// be dropped — and with the block tint gone they were then shown nowhere at all. #[test] - fn tips_too_narrow_to_label_are_dropped_and_counted() { + fn crowded_men_wrap_into_rows_rather_than_being_dropped() { let crowd: Vec = (0..40).map(|_| origin(2, "Cork, Co. Cork, Ireland")).collect(); let laid = layout(&tree(), &crowd, Level::Admin, 40); - assert!(laid.tips.is_empty(), "40 men cannot each hold a legible box"); - assert_eq!(laid.tips_suppressed, 40, "reported, never silent"); - // They are still fully present in the composition. - assert_eq!(laid.bands.iter().find(|b| b.id == 2).unwrap().with_origin, 40); - // Every tip that IS drawn is wide enough to letter. - let few = layout(&tree(), &[origin(2, "Cork, Co. Cork, Ireland")], Level::Admin, 1); - assert!(few.tips.iter().all(|t| t.w >= MIN_TIP_W)); - assert_eq!(few.tips_suppressed, 0); + assert_eq!(laid.tips.len(), 40, "every man gets a box"); + assert!(laid.tips.iter().all(|t| t.w >= MIN_TIP_W), "and every box can hold a label"); + // They stack: more than one row, and rows are a tip-height apart. + let rows: std::collections::BTreeSet = laid.tips.iter().map(|t| t.y as i64).collect(); + assert!(rows.len() > 1, "40 men do not fit one row under a leaf block"); + // The canvas grew to hold them — nothing is drawn outside it. + assert!(laid.tips.iter().all(|t| t.y + t.h <= laid.height + 0.01)); } /// A sample placed deeper than the walk still has to land somewhere, or the chart quietly diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html index ba0ce83d..e8286105 100644 --- a/rust/crates/du-web/templates/tree/origins.html +++ b/rust/crates/du-web/templates/tree/origins.html @@ -65,12 +65,6 @@

{{ t.get("tree.origins.title") }} · {{ name }}

{% if l.unresolved > 0 %} · {{ l.unresolved }} {{ t.get("tree.origins.unresolved") }} {% endif %} - {% if l.pruned > 0 %} - · {{ l.pruned }} {{ t.get("tree.origins.pruned") }} - {% endif %} - {% if l.tips_suppressed > 0 %} - · {{ l.tips_suppressed }} {{ t.get("tree.origins.tipshidden") }} - {% endif %}

{% if l.legend.len() >= 2 %} @@ -90,14 +84,6 @@

{{ t.get("tree.origins.title") }} · {{ name }}

- - {# An unmeasured branch is hatched so it can never read as a measured short one. #} - - - - - - {# Mutation ruler, walked down the deepest lineage. Mutations accrue at a roughly steady rate — measured on this tree, branch length tracks SNP count at r=0.975, ~69 years each — so the axis reads as elapsed time without depending on a per-branch age estimate. #} @@ -111,14 +97,10 @@

{{ t.get("tree.origins.title") }} · {{ name }}

>{{ t.get("tree.origins.axis") }} {% for b in l.bands %} - + - {% if !b.dated %} - - {% endif %} {{ b.name }} — {{ b.snp_total }} {{ t.get("tree.origins.snps") }} · {{ b.with_origin }} {{ t.get("tree.origins.with") }}, {{ b.without_origin }} {{ t.get("tree.origins.without") }}{% if b.has_more %} · {{ t.get("tree.origins.more") }}{% endif %} {# Direct label: the relief the light-mode contrast warning obliges. Text wears text tokens, diff --git a/rust/locales/en.txt b/rust/locales/en.txt index 35a0072a..9aa33e76 100644 --- a/rust/locales/en.txt +++ b/rust/locales/en.txt @@ -179,8 +179,6 @@ tree.origins.back=Back to tree tree.origins.tmrca=Clade TMRCA tree.origins.placed=placed samples tree.origins.unresolved=with no published origin -tree.origins.pruned=branches hidden (no published origin below them) -tree.origins.tipshidden=men shown in the composition only (too narrow to label) tree.origins.more=has further branches below — click to open tree.origins.snps=equivalent SNPs tree.origins.axis=SNPs diff --git a/rust/locales/es.txt b/rust/locales/es.txt index 25ba7d09..83562d2b 100644 --- a/rust/locales/es.txt +++ b/rust/locales/es.txt @@ -131,8 +131,6 @@ tree.origins.back=Volver al árbol tree.origins.tmrca=TMRCA del clado tree.origins.placed=muestras situadas tree.origins.unresolved=sin origen publicado -tree.origins.pruned=ramas ocultas (sin origen publicado debajo) -tree.origins.tipshidden=hombres mostrados solo en la composición (demasiado estrecho para etiquetar) tree.origins.more=tiene más ramas debajo — pulse para abrir tree.origins.snps=SNP equivalentes tree.origins.axis=SNP diff --git a/rust/locales/fr.txt b/rust/locales/fr.txt index dc3bd739..9fdd81eb 100644 --- a/rust/locales/fr.txt +++ b/rust/locales/fr.txt @@ -131,8 +131,6 @@ tree.origins.back=Retour à l'arbre tree.origins.tmrca=TMRCA du clade tree.origins.placed=échantillons placés tree.origins.unresolved=sans origine publiée -tree.origins.pruned=branches masquées (aucune origine publiée en dessous) -tree.origins.tipshidden=hommes indiqués seulement dans la composition (trop étroit pour étiqueter) tree.origins.more=comporte d’autres branches en dessous — cliquez pour ouvrir tree.origins.snps=SNP équivalents tree.origins.axis=SNP From b05e720ebaf25405ceb7e7839dac2e7e5a12e60a Mon Sep 17 00:00:00 2001 From: James Kane Date: Fri, 7 Aug 2026 06:40:11 -0500 Subject: [PATCH 09/10] fix(tree): the icicle follows the page's theme, and its type is readable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two problems, one of them mine to begin with. THE BLOCK HEADERS WERE ILLEGIBLE ON A DARK-MODE DESKTOP. The icicle carried a `prefers-color-scheme: dark` block, but the site itself is light-only — so on a dark OS the SVG surface went dark while the Bootstrap-derived block fills stayed light, and every branch name became dark text with a dark halo on a light block. Keying off the OS was wrong in the first place: a chart embedded in a page has to follow that page. The dark values now hang off `[data-bs-theme="dark"]` alone, which is what the app would set if it ever gains a dark theme, and `--surface-1` defaults to `--bs-body-bg` so it tracks whatever the page is. The label halo also now uses the block fill rather than the canvas, since the block is what sits behind the text. THE TYPE WAS TOO SMALL — 9px SNP names, 10px branch names. Names go to 12px bold, SNPs and men's labels to 11px, the ruler to 10px, and the geometry is now derived from those sizes rather than hard-coded beside them: line heights, tip height, minimum tip width and the leaf block width all follow the font constants, so changing the type cannot silently break the layout again. A leaf block widens 74px → 90px, which is what a 12-character SNP name (`14405732-C-T`) actually needs at 11px. R-DF85 at depth 4 costs 3,576px → 4,296px of width for it. That is the trade: legible at normal viewing distance instead of technically-present. Co-Authored-By: Claude Opus 5 (1M context) --- rust/crates/du-web/assets/main.css | 38 ++++++++----------- rust/crates/du-web/src/origins_layout.rs | 37 +++++++++++------- .../crates/du-web/templates/tree/origins.html | 12 +++--- 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/rust/crates/du-web/assets/main.css b/rust/crates/du-web/assets/main.css index c6c512a6..87c89664 100644 --- a/rust/crates/du-web/assets/main.css +++ b/rust/crates/du-web/assets/main.css @@ -175,34 +175,26 @@ code { color: #495057; background-color: #f8f9fa; } } /* ── Ancestral-origin icicle (proposals/ancestral-origin-icicle.md §5) ───────── */ -/* Colour roles live here so light/dark swap in one place and the SVG is written - against slots rather than raw hex. The palette is the validated 8-slot - categorical set — fixed order, never cycled. Slot 0 is the reserved neutral - carrying both "Other" and "no locality recorded": an absence is not an - identity, so it must not wear a categorical hue. - - Validated with the data-viz palette checker in both modes. Light mode raises a - contrast warning on three slots (aqua/yellow/magenta below 3:1 on a light - surface), which obliges relief — hence the always-on band labels and the - table view in origins.html. */ +/* Colour roles live here so the chart is written against roles rather than raw hex. + The palette is the validated 8-slot categorical set — fixed order, never cycled. + Slot 0 is the reserved neutral carrying both "Other" and "no locality recorded": + an absence is not an identity, so it must not wear a categorical hue. + + THEME FOLLOWS THE PAGE, NOT THE OS. An earlier cut keyed the dark values off + `prefers-color-scheme`, but the site itself is light-only — so on a dark-mode + desktop the SVG surface went dark while the Bootstrap-derived block fills stayed + light, and every block label became dark text with a dark halo on a light block. + The dark values now hang off `[data-bs-theme="dark"]` alone, which is what the + app would set if it ever gains a dark theme. */ .origins-icicle { - --surface-1: #ffffff; + --surface-1: var(--bs-body-bg, #ffffff); --o-0: #adb5bd; /* reserved neutral: Other / no locality */ --o-1: #2a78d6; --o-2: #eb6834; --o-3: #1baf7a; --o-4: #eda100; --o-5: #e87ba4; --o-6: #008300; --o-7: #4a3aa7; --o-8: #e34948; display: block; background: var(--surface-1); } -@media (prefers-color-scheme: dark) { - .origins-icicle { - --surface-1: #1a1a19; - --o-0: #6c757d; - --o-1: #3987e5; --o-2: #d95926; --o-3: #199e70; --o-4: #c98500; - --o-5: #d55181; --o-6: #008300; --o-7: #9085e9; --o-8: #e66767; - } -} [data-bs-theme="dark"] .origins-icicle { - --surface-1: #1a1a19; --o-0: #6c757d; --o-1: #3987e5; --o-2: #d95926; --o-3: #199e70; --o-4: #c98500; --o-5: #d55181; --o-6: #008300; --o-7: #9085e9; --o-8: #e66767; @@ -215,8 +207,8 @@ code { color: #495057; background-color: #f8f9fa; } .origins-band a { cursor: pointer; } .origins-band:hover .origins-band-bg { stroke: var(--bs-primary, #0d6efd); stroke-width: 1.5; } /* Text wears text tokens, never the series colour. */ -.origins-band-label { fill: var(--bs-body-color, #212529); font-weight: 600; paint-order: stroke; stroke: var(--surface-1); stroke-width: 2.5px; } -.origins-tip-label { fill: var(--bs-secondary-color, #6c757d); paint-order: stroke; stroke: var(--surface-1); stroke-width: 2.5px; } +.origins-band-label { fill: var(--bs-body-color, #212529); font-weight: 600; paint-order: stroke; stroke: var(--bs-tertiary-bg, #f8f9fa); stroke-width: 3px; } +.origins-tip-label { fill: var(--bs-body-color, #212529); paint-order: stroke; stroke: var(--surface-1); stroke-width: 3px; } /* Stacked segments are separated by surface, not by a stroke (mark spec). */ .origins-seg { stroke: none; } @@ -255,6 +247,6 @@ code { color: #495057; background-color: #f8f9fa; } .origins-more { fill: var(--bs-secondary-color, #6c757d); font-weight: 700; } /* Block contents: the branch's equivalent SNPs. Recessive against the block, and never coloured by anything — a mutation has no locality. */ -.origins-snp { fill: var(--bs-body-color, #212529); opacity: .75; font-family: var(--bs-font-monospace, monospace); } +.origins-snp { fill: var(--bs-body-color, #212529); opacity: .8; font-family: var(--bs-font-monospace, monospace); } .origins-snp-more { fill: var(--bs-secondary-color, #6c757d); font-style: italic; } .origins-tick-unit { fill: var(--bs-secondary-color, #6c757d); font-style: italic; } diff --git a/rust/crates/du-web/src/origins_layout.rs b/rust/crates/du-web/src/origins_layout.rs index afc68f78..6966aa99 100644 --- a/rust/crates/du-web/src/origins_layout.rs +++ b/rust/crates/du-web/src/origins_layout.rs @@ -34,27 +34,36 @@ use du_db::origins::SampleOrigin; use du_db::place::Level; use std::collections::HashMap; -/// Canvas geometry at scale 1. -const LEAF_W: f64 = 74.0; +// Type sizes. These were 9-10px and unreadable at normal viewing distance; everything below is +// sized off them so the geometry follows the type rather than the other way round. +/// Branch name at the top of a block. +pub(crate) const NAME_FONT: f64 = 12.0; +/// SNP names inside a block, and a man's label. +pub(crate) const SNP_FONT: f64 = 11.0; +pub(crate) const TIP_FONT: f64 = 11.0; + +/// Canvas geometry at scale 1. Wide enough for a 12-character SNP name at [`SNP_FONT`] +/// (`14405732-C-T`) plus padding — the narrowest a block can be and still letter its contents. +const LEAF_W: f64 = 90.0; const H_GAP: f64 = 4.0; /// Ruler graduation interval, in mutations. const TICK_SNPS: usize = 5; /// Sample tips hang in a band below the youngest branch. -const TIP_H: f64 = 16.0; +const TIP_H: f64 = 19.0; /// Narrowest a man's box may be and still carry a readable label. Men wrap into further rows /// rather than being packed below it. -const MIN_TIP_W: f64 = 26.0; +const MIN_TIP_W: f64 = 30.0; /// Vertical gap between wrapped rows of men. const TIP_ROW_GAP: f64 = 2.0; const TIP_GAP: f64 = 10.0; const GUTTER_W: f64 = 54.0; const MARGIN: f64 = 8.0; /// One line of SNP text inside a block. -const SNP_LINE_H: f64 = 11.0; +const SNP_LINE_H: f64 = 14.0; /// Padding inside a block before its SNP list starts. const SNP_PAD: f64 = 4.0; /// The branch-name line at the top of every block. -const NAME_LINE_H: f64 = 12.0; +const NAME_LINE_H: f64 = 16.0; /// Categorical slots available before folding into "Other". The palette is fixed-order and never /// cycled; a ninth locality is not given a generated hue. @@ -452,7 +461,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl let snps = flow_snps(&n.snps, left[i], y, extent[i]); bands.push(Band { id: n.id, - label: fit(&n.name, extent[i], 10.0), + label: fit(&n.name, extent[i], NAME_FONT), name: n.name.clone(), x: left[i], y, @@ -464,7 +473,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl without_origin, snps, snp_total: n.snps.len(), - cramped: h < 14.0, + cramped: h < NAME_LINE_H, has_more: has_more.contains(&n.id), }); deepest = deepest.max(y + h); @@ -513,7 +522,7 @@ pub fn layout(all_nodes: &[Node], all_origins: &[SampleOrigin], level: Level, pl tips.push(Tip { slot: locality.and_then(|l| slots.get(l).copied()).unwrap_or(0), full: label.clone(), - label: fit(&label, w, 9.0), + label: fit(&label, w, TIP_FONT), x: bx + (k % cols) as f64 * (w + H_GAP), y: tip_y + (k / cols) as f64 * (TIP_H + TIP_ROW_GAP), w, @@ -561,7 +570,7 @@ fn flow_snps(names: &[String], x: f64, y: f64, w: f64) -> Vec { .iter() .enumerate() .map(|(k, name)| SnpCell { - name: fit(name, w - 2.0 * SNP_PAD, 9.0), + name: fit(name, w - 2.0 * SNP_PAD, SNP_FONT), x: x + SNP_PAD, y: top + (k as f64 + 0.8) * SNP_LINE_H, }) @@ -1015,10 +1024,10 @@ mod tests { /// was unreadable — which the rectangle-only layout assertions could never have caught. #[test] fn labels_are_fitted_to_their_boxes() { - assert_eq!(fit("Kane", 74.0, 9.0), "Kane", "what fits is left alone"); - let cut = fit("Sullivan · Co. Limerick", 74.0, 9.0); + assert_eq!(fit("Kane", LEAF_W, SNP_FONT), "Kane", "what fits is left alone"); + let cut = fit("Sullivan · Co. Limerick", LEAF_W, SNP_FONT); assert!(cut.ends_with('…') && cut.chars().count() < 23); - assert!(fit("anything", 4.0, 9.0).is_empty(), "no room at all yields no text"); + assert!(fit("anything", 4.0, SNP_FONT).is_empty(), "no room at all yields no text"); let laid = layout(&tree(), &[origin(2, "Kenmare, Co. Kerry, Ireland")], Level::Admin, 1); let tip = laid.tips.first().expect("one man"); @@ -1026,7 +1035,7 @@ mod tests { assert!(tip.label.chars().count() <= tip.full.chars().count()); // Every band's drawn label fits the band it sits in. for b in &laid.bands { - assert!((b.label.chars().count() as f64) * 10.0 * CHAR_W_RATIO <= b.w, "{}", b.name); + assert!((b.label.chars().count() as f64) * NAME_FONT * CHAR_W_RATIO <= b.w, "{}", b.name); } } diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html index e8286105..b48fc031 100644 --- a/rust/crates/du-web/templates/tree/origins.html +++ b/rust/crates/du-web/templates/tree/origins.html @@ -90,10 +90,10 @@

{{ t.get("tree.origins.title") }} · {{ name }}

{% for tk in l.ticks %} - {{ tk.snps }} + {{ tk.snps }} {% endfor %} - {{ t.get("tree.origins.axis") }} {% for b in l.bands %} @@ -106,17 +106,17 @@

{{ t.get("tree.origins.title") }} · {{ name }}

{# Direct label: the relief the light-mode contrast warning obliges. Text wears text tokens, never the series colour. #} {% if !b.cramped %} - {{ b.label }} + {{ b.label }} {% endif %} {# Branches were folded into this band. Marked so "simple" is never confused with "not shown" — click through to draw them. #} {# The branch's equivalent SNPs — the mutations are unordered, so the list IS the block. #} {% for sn in b.snps %} - {{ sn.name }} + {{ sn.name }} {% endfor %} {% if b.has_more %} + + font-size="11" class="origins-more">+ {% endif %} {% endfor %} @@ -126,7 +126,7 @@

{{ t.get("tree.origins.title") }} · {{ name }}

{{ tp.full }} - {{ tp.label }} + {{ tp.label }} {% endfor %} From 9d5365cb6d6af99a8273ab33fbd1f52773f2393b Mon Sep 17 00:00:00 2001 From: James Kane Date: Fri, 7 Aug 2026 10:42:20 -0500 Subject: [PATCH 10/10] feat(tree): put the ancestral-origin icicle in the site's navigation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The view existed but was reachable only by typing its URL. Three ways in now, each at the altitude it belongs to. PER CLADE — a button in the tree's SNP sidebar, beside the sample-map panel that answers the neighbouring question. It navigates rather than swapping a fragment: the icicle is a whole view, not a sidebar widget. PER LINEAGE — `/ytree/origins` and `/mtree/origins`, the Tools menu entry. These open on the tree's default root, which is always older than the era gate, so they render the signpost — and that turns out to be the right landing page rather than a dead end: it states the constraint and immediately offers the clades young enough to have origins. For that to be useful the signpost had to stop listing whatever the tree walk happened to return. It now ranks by how many men a clade holds and drops the ones holding none, so `/ytree/origins` opens on R-DF85 (283), R-S673 (239), R-S764 (232) rather than the alphabetically-first branches. 5,113 clades qualify under the Y root; 60 are shown and the page says so, because a capped list that looks complete is worse than one that admits its cap. Each entry carries its sample count and TMRCA, which is what a reader picks on. Co-Authored-By: Claude Opus 5 (1M context) --- rust/crates/du-web/src/routes/tree.rs | 78 +++++++++++++++++-- rust/crates/du-web/templates/base.html | 1 + .../crates/du-web/templates/tree/origins.html | 14 +++- .../du-web/templates/tree/snp_sidebar.html | 6 ++ rust/locales/en.txt | 4 + rust/locales/es.txt | 4 + rust/locales/fr.txt | 4 + 7 files changed, 101 insertions(+), 10 deletions(-) diff --git a/rust/crates/du-web/src/routes/tree.rs b/rust/crates/du-web/src/routes/tree.rs index 0dbed2b8..b53e0a4c 100644 --- a/rust/crates/du-web/src/routes/tree.rs +++ b/rust/crates/du-web/src/routes/tree.rs @@ -54,6 +54,11 @@ pub fn router() -> Router { .route("/ytree/node/:name/geo-data", get(y_clade_geo_data)) .route("/mtree/node/:name/geo-data", get(mt_clade_geo_data)) // Genealogical-era ancestral-origin icicle (proposals/ancestral-origin-icicle.md). + // The lineage-level route is the nav entry point: it opens on the tree root, which is + // always older than the era gate, so it renders the signpost — the clades young enough to + // have origins, ranked by how many men they hold. + .route("/ytree/origins", get(y_origins_index)) + .route("/mtree/origins", get(mt_origins_index)) .route("/ytree/node/:name/origins", get(y_origins)) .route("/mtree/node/:name/origins", get(mt_origins)) // Curator triage for sample leaves whose published call didn't resolve to a node. @@ -160,6 +165,8 @@ struct SnpSidebar { name: String, /// URL of this node's sample-map panel fragment (lazy-loaded on click). geo_href: String, + /// URL of this node's ancestral-origins icicle (a full page, not a panel). + origins_href: String, provenance: Option, /// The consolidated age block: the branch TMRCA/formed on a time axis (with any /// ancient-DNA anchors). Replaces the old textual formed/TMRCA provenance rows. @@ -459,10 +466,16 @@ async fn snp_sidebar( base_path_for(dna_type), utf8_percent_encode(&name, NON_ALPHANUMERIC) ); + let origins_href = format!( + "{}/node/{}/origins", + base_path_for(dna_type), + utf8_percent_encode(&name, NON_ALPHANUMERIC) + ); Ok(html(&SnpSidebar { t: locale.t, name, geo_href, + origins_href, provenance, age, variants, @@ -538,6 +551,17 @@ const ORIGINS_DEPTH_DEFAULT: i32 = 4; const ORIGINS_DEPTH_MIN: i32 = 1; const ORIGINS_DEPTH_MAX: i32 = 8; const ORIGINS_DEPTH_OPTIONS: [i32; 6] = [2, 3, 4, 5, 6, 8]; +/// Clades offered on the signpost when the requested one is older than the era gate. Under the Y +/// root thousands qualify; these are the largest, and the total is shown beside them. +const ORIGINS_SIGNPOST_CAP: usize = 60; + +/// A clade young enough to have ancestral origins, offered as a way in. +struct EligibleClade { + name: String, + href: String, + samples: i64, + tmrca_ybp: i32, +} #[derive(Deserialize)] struct OriginsQuery { @@ -555,6 +579,27 @@ fn origins_level(s: Option<&str>) -> place::Level { } } +/// Lineage-level entry: the origins view rooted at the tree's default root. +async fn y_origins_index( + st: State, + locale: Locale, + user: crate::auth::MaybeUser, + q: Query, +) -> Result { + let root = default_root_name(&st.pool, DnaType::YDna, "Y").await?; + origins(st, locale, user, Path(root), q, DnaType::YDna).await +} + +async fn mt_origins_index( + st: State, + locale: Locale, + user: crate::auth::MaybeUser, + q: Query, +) -> Result { + let root = default_root_name(&st.pool, DnaType::MtDna, "L").await?; + origins(st, locale, user, Path(root), q, DnaType::MtDna).await +} + async fn y_origins( st: State, locale: Locale, @@ -602,13 +647,27 @@ async fn origins( let too_old = node.tmrca_ybp.is_none_or(|t| t > ORIGINS_MAX_YBP); if too_old { let window = du_db::haplogroup::subtree_window(&st.pool, dna_type, &name, ORIGINS_WALK).await?; - let eligible: Vec = window + // Rank the eligible clades by how many men they hold, so the entry point opens on the + // branches worth looking at rather than the alphabetically-first ones. Under the Y root + // there are thousands; the cap is stated rather than silently applied. + let counts = du_db::tree_sample::cumulative_counts(&st.pool, dna_type).await?; + let mut ranked: Vec<(String, i64, i32)> = window .iter() .filter(|n| n.id != node.id.0 && n.tmrca_ybp.is_some_and(|t| t <= ORIGINS_MAX_YBP)) - .filter(|n| !is_private_node(&n.name)) - .map(|n| Crumb { - href: format!("{base_path}/node/{}/origins", encode(&n.name)), - name: n.name.clone(), + .filter(|n| !is_private_node(&n.name) && !is_uuid_label(&n.name)) + .map(|n| (n.name.clone(), counts.get(&n.id).copied().unwrap_or(0), n.tmrca_ybp.unwrap_or(0))) + .filter(|(_, samples, _)| *samples > 0) + .collect(); + ranked.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + let eligible_total = ranked.len(); + ranked.truncate(ORIGINS_SIGNPOST_CAP); + let eligible: Vec = ranked + .into_iter() + .map(|(name, samples, tmrca_ybp)| EligibleClade { + href: format!("{base_path}/node/{}/origins", encode(&name)), + name, + samples, + tmrca_ybp, }) .collect(); let page = OriginsPageTemplate { @@ -625,6 +684,7 @@ async fn origins( crumbs, laid: None, eligible, + eligible_total, placed: 0, }; return Ok(html(&page)); @@ -679,6 +739,7 @@ async fn origins( crumbs, laid: Some(laid), eligible: Vec::new(), + eligible_total: 0, placed: placed.max(0), })) } @@ -713,8 +774,11 @@ struct OriginsPageTemplate { crumbs: Vec, /// `None` when the clade is older than the era gate — the template shows `eligible` instead. laid: Option, - /// Descendant clades that *are* inside the era, offered when this one is not. - eligible: Vec, + /// Descendant clades that *are* inside the era, offered when this one is not — ranked by how + /// many men they hold, capped at [`ORIGINS_SIGNPOST_CAP`]. + eligible: Vec, + /// How many were eligible before the cap, so the truncation is never silent. + eligible_total: usize, /// Placed samples under the clade — the denominator the composition is reported against. placed: i64, } diff --git a/rust/crates/du-web/templates/base.html b/rust/crates/du-web/templates/base.html index bc120e7f..7565f4f0 100644 --- a/rust/crates/du-web/templates/base.html +++ b/rust/crates/du-web/templates/base.html @@ -35,6 +35,7 @@
  • {{ t.get("nav.variants") }}
  • {{ t.get("nav.coverage") }}
  • {{ t.get("nav.strMarkers") }}
    • +
  • {{ t.get("nav.origins") }}
    • {# Full page load (not boosted) so Leaflet's scripts/styles initialize. #}
  • {{ t.get("nav.map") }}
    • diff --git a/rust/crates/du-web/templates/tree/origins.html b/rust/crates/du-web/templates/tree/origins.html index b48fc031..0a5675a3 100644 --- a/rust/crates/du-web/templates/tree/origins.html +++ b/rust/crates/du-web/templates/tree/origins.html @@ -168,10 +168,18 @@

    {{ t.get("tree.origins.title") }} · {{ name }}

    {% if eligible.is_empty() %}

    {{ t.get("tree.origins.nonebelow") }}

    {% else %} -

    {{ t.get("tree.origins.trybelow") }}

    -
    +

    + {{ t.get("tree.origins.trybelow") }} + {% if eligible_total > eligible.len() %} + ({{ t.get("tree.origins.showing") }} {{ eligible.len() }} {{ t.get("tree.origins.of") }} {{ eligible_total }}) + {% endif %} +

    +
    {% for c in eligible %} - {{ c.name }} + + {{ c.name }} + {{ c.samples }} · {{ c.tmrca_ybp }} {{ t.get("tree.geo.axis") }} + {% endfor %}
    {% endif %} diff --git a/rust/crates/du-web/templates/tree/snp_sidebar.html b/rust/crates/du-web/templates/tree/snp_sidebar.html index cf39ddb6..da849540 100644 --- a/rust/crates/du-web/templates/tree/snp_sidebar.html +++ b/rust/crates/du-web/templates/tree/snp_sidebar.html @@ -142,6 +142,12 @@
    + + {# The genealogical-era origins icicle for this clade. A full page rather than a panel: it is + a whole view, not a sidebar widget. #} + + {{ t.get("tree.origins.open") }} +
    {% endif %} diff --git a/rust/locales/en.txt b/rust/locales/en.txt index 9aa33e76..6c44b357 100644 --- a/rust/locales/en.txt +++ b/rust/locales/en.txt @@ -8,6 +8,7 @@ nav.references=References nav.map=Map nav.coverage=Coverage nav.strMarkers=Y-STR Markers +nav.origins=Ancestral origins nav.curator=Curator nav.tools=Tools nav.profile=Profile @@ -171,6 +172,7 @@ tree.geo.loading=Loading map… # Ancestral-origin icicle (proposals/ancestral-origin-icicle.md) tree.origins.title=Ancestral origins +tree.origins.open=Ancestral origins tree.origins.level=Detail level tree.origins.level.country=Country tree.origins.level.admin=County / State @@ -192,6 +194,8 @@ tree.origins.col.count=Samples tree.origins.tooold=Ancestral origins are only meaningful in the genealogical era. This clade is older than that, so its branches would each aggregate to a whole continent. tree.origins.ceiling=cutoff tree.origins.trybelow=Try one of these younger branches below it: +tree.origins.showing=showing +tree.origins.of=of tree.origins.nonebelow=No branch below this clade is young enough yet. tree.geo.tmrca=TMRCA tree.geo.formed=Formed diff --git a/rust/locales/es.txt b/rust/locales/es.txt index 83562d2b..13f2d7e0 100644 --- a/rust/locales/es.txt +++ b/rust/locales/es.txt @@ -8,6 +8,7 @@ nav.references=Referencias nav.map=Mapa nav.coverage=Cobertura nav.strMarkers=Marcadores Y-STR +nav.origins=Orígenes ancestrales nav.curator=Curador nav.tools=Herramientas nav.profile=Perfil @@ -123,6 +124,7 @@ tree.geo.loading=Cargando mapa… # Ancestral-origin icicle (proposals/ancestral-origin-icicle.md) tree.origins.title=Orígenes ancestrales +tree.origins.open=Orígenes ancestrales tree.origins.level=Nivel de detalle tree.origins.level.country=País tree.origins.level.admin=Condado / Estado @@ -144,6 +146,8 @@ tree.origins.col.count=Muestras tree.origins.tooold=Los orígenes ancestrales solo tienen sentido en la era genealógica. Este clado es más antiguo, así que cada una de sus ramas se agregaría a un continente entero. tree.origins.ceiling=límite tree.origins.trybelow=Pruebe una de estas ramas más recientes: +tree.origins.showing=mostrando +tree.origins.of=de tree.origins.nonebelow=Todavía no hay ninguna rama lo bastante reciente bajo este clado. tree.geo.tmrca=TMRCA tree.geo.formed=Formado diff --git a/rust/locales/fr.txt b/rust/locales/fr.txt index 9fdd81eb..39dc7739 100644 --- a/rust/locales/fr.txt +++ b/rust/locales/fr.txt @@ -8,6 +8,7 @@ nav.references=Références nav.map=Carte nav.coverage=Couverture nav.strMarkers=Marqueurs Y-STR +nav.origins=Origines ancestrales nav.curator=Curateur nav.tools=Outils nav.profile=Profil @@ -123,6 +124,7 @@ tree.geo.loading=Chargement de la carte… # Ancestral-origin icicle (proposals/ancestral-origin-icicle.md) tree.origins.title=Origines ancestrales +tree.origins.open=Origines ancestrales tree.origins.level=Niveau de détail tree.origins.level.country=Pays tree.origins.level.admin=Comté / État @@ -144,6 +146,8 @@ tree.origins.col.count=Échantillons tree.origins.tooold=Les origines ancestrales n'ont de sens que dans l'ère généalogique. Ce clade est plus ancien : chacune de ses branches se ramènerait à un continent entier. tree.origins.ceiling=seuil tree.origins.trybelow=Essayez l'une de ces branches plus récentes : +tree.origins.showing=affichage de +tree.origins.of=sur tree.origins.nonebelow=Aucune branche sous ce clade n'est encore assez récente. tree.geo.tmrca=TMRCA tree.geo.formed=Formé