From 5a77a654f3c5f629079c3c0572e3ec86808a6f03 Mon Sep 17 00:00:00 2001 From: David de Boer Date: Mon, 31 Aug 2026 15:41:56 +0200 Subject: [PATCH] =?UTF-8?q?fix(search):=20sort=20a=20document=20that=20has?= =?UTF-8?q?=20no=20title=20in=20the=20reader=E2=80=99s=20language?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fall back, per locale, to the document’s first declared-locale value when it carries no value in that language, so every sort key a query can name is populated whenever the field holds anything at all - leave the search companions per-locale: a query fans out over every locale key at once, so only the sort – which names a single key – needs the fallback - document the rule and the reason under Locales in the search reference --- docs/reference/search.md | 15 ++++++-- .../registry-extraction.integration.test.ts | 4 ++ packages/search/src/project.ts | 37 ++++++++++++++----- packages/search/test/project.test.ts | 18 +++++++++ 4 files changed, 62 insertions(+), 12 deletions(-) diff --git a/docs/reference/search.md b/docs/reference/search.md index 022c2430..7d84f22b 100644 --- a/docs/reference/search.md +++ b/docs/reference/search.md @@ -952,15 +952,24 @@ language present: lets a language that needs a dedicated tokenizer set its own stemming `locale` in the engine schema); - `sortable` → `title_sort_nl`/`title_sort_en` (folded, so a locale-switching UI - sorts on the active language). + sorts on the active language). Each locale’s key **falls back** to the + document’s first value in `locales` order when that language is absent, so a + document titled only in English still sorts under a title in a Dutch request. + A sort names a single key: without the fallback every document lacking the + active language would tie on the empty string and come back in relevance + order, which is what a collection of untagged titles looks like when it is + “sorted” by name. Search needs no such fallback – a query fans out over every + locale key at once. A field with `searchable` but no `output` is **search-only** – folded and stemmed for retrieval but never rendered (e.g. a creator searched here but shown via a separate label). **Only listed locales are indexed** (searched and sorted); a literal whose language tag is not in `locales` is still **displayed** but not matched or sorted on. Display fields are **omitted, never empty**, when a document -lacks that language, and the per-locale search/sort fields likewise, so declare -them optional in the engine schema and sort with `missing_values: last`. A +lacks that language, and the per-locale search fields likewise, so declare them +optional in the engine schema and sort with `missing_values: last`. A sort field +is present whenever the field holds any value in a declared locale (the fallback +above); it is absent only when the field is. A deployment that wants to bound the displayed languages narrows them upstream (e.g. selecting a language subset in its CONSTRUCT query), since preservation is the default. diff --git a/packages/search-pipeline/test/registry-extraction.integration.test.ts b/packages/search-pipeline/test/registry-extraction.integration.test.ts index a7cf0b37..638d5d7e 100644 --- a/packages/search-pipeline/test/registry-extraction.integration.test.ts +++ b/packages/search-pipeline/test/registry-extraction.integration.test.ts @@ -160,6 +160,10 @@ describe('registry-sourced extraction: scoped to the dataset’s own graph', () label_sort_nl: 'vaandels van limburg', label_search_en: 'banners of limburg', label_sort_en: 'banners of limburg', + // The `und` locale is declared but untagged labels are absent here, so + // its sort key falls back to the first declared locale’s value – a + // reader whose language the request does not state still gets an order. + label_sort_und: 'vaandels van limburg', description_nl: 'Een dataset over vaandels.', description_search_nl: 'een dataset over vaandels.', publisher: ['https://ex/org/trace'], diff --git a/packages/search/src/project.ts b/packages/search/src/project.ts index 8e982318..1a4e32ee 100644 --- a/packages/search/src/project.ts +++ b/packages/search/src/project.ts @@ -455,6 +455,17 @@ function referenceValues( * `sortable`) stay on the declared `locales`, which drive the indexed, stemmed, * weighted fanout; a value in an undeclared language is not indexed. Absent * languages emit nothing. + * + * The two companions differ in what an absent language means. A search query + * fans out over *every* locale key at once, so a document titled in one + * language is found whichever language the reader asks in. A sort names a + * **single** key, so a locale key left empty is not “no value in Dutch” – it is + * the empty string, which ties with every other document missing that language + * and leaves them in relevance order. Each locale’s sort key therefore falls + * back to the document’s first value in `locales` order: a collection of + * untagged titles sorts by title in a Dutch request rather than not at all. + * Ordering across languages is approximate by nature; a total order over the + * titles a reader actually sees beats a partial one over the tagged few. */ function applyText( document: ProjectedNode, @@ -476,22 +487,30 @@ function applyText( // here it simply indexes nothing. if (field.searchable !== undefined || field.sortable === true) { const names = physicalFields(field); - field.locales.forEach((locale, index) => { - const localeValues = values + const valuesPerLocale = field.locales.map((locale) => + values .filter((value) => value.lang === locale) - .map((value) => value.value); - if (localeValues.length === 0) { - return; - } - if (field.searchable) { + .map((value) => value.value), + ); + // What every locale’s sort key falls back to: the document’s first value + // in `locales` order, which is the declaration’s own statement of which + // language stands in for the others. + const fallbackSortValue = valuesPerLocale.find( + (localeValues) => localeValues.length > 0, + )?.[0]; + valuesPerLocale.forEach((localeValues, index) => { + if (field.searchable !== undefined && localeValues.length > 0) { setString( document, names.search[index], foldedSearchValue(localeValues), ); } - if (field.sortable) { - setString(document, names.sort[index], fold(localeValues[0])); + if (field.sortable === true) { + const sortValue = localeValues[0] ?? fallbackSortValue; + if (sortValue !== undefined) { + setString(document, names.sort[index], fold(sortValue)); + } } }); } diff --git a/packages/search/test/project.test.ts b/packages/search/test/project.test.ts index d2aa805d..5f4f50de 100644 --- a/packages/search/test/project.test.ts +++ b/packages/search/test/project.test.ts @@ -351,6 +351,24 @@ describe('projectDocument', () => { expect(document.title_sort_nl).toBeUndefined(); }); + it('falls back to the document’s first value for a locale it has no title in', () => { + const document = projectDocument( + { + '@id': 'https://ex/d/5b', + [dsKey('title')]: { '@language': 'en', '@value': 'Title' }, + }, + { name: 'Dataset', class: DATASET, fields }, + ); + // A sort names one key, so an empty `title_sort_nl` would tie this document + // with every other one lacking a Dutch title – no order at all. It sorts + // under its English title instead. Search is unaffected: that query fans + // out over every locale key, so the fallback would only duplicate. + expect(document.title_sort_nl).toBe('title'); + expect(document.title_sort_en).toBe('title'); + expect(document.title_search_nl).toBeUndefined(); + expect(document.title_search_en).toBe('title'); + }); + it('displays a value whose language is outside locales, but does not index it', () => { const document = projectDocument( {