fix(search): sort a document that has no title in the reader’s language

docs/reference/search.md

@@ -952,15 +952,24 @@ language present:
   lets a language that needs a dedicated tokenizer set its own stemming `locale`
   in the engine schema);
 - `sortable` → `title_sort_nl`/`title_sort_en` (folded, so a locale-switching UI
-  sorts on the active language).
+  sorts on the active language). Each locale’s key **falls back** to the
+  document’s first value in `locales` order when that language is absent, so a
+  document titled only in English still sorts under a title in a Dutch request.
+  A sort names a single key: without the fallback every document lacking the
+  active language would tie on the empty string and come back in relevance
+  order, which is what a collection of untagged titles looks like when it is
+  “sorted” by name. Search needs no such fallback – a query fans out over every
+  locale key at once.
 
 A field with `searchable` but no `output` is **search-only** – folded and stemmed
 for retrieval but never rendered (e.g. a creator searched here but shown via a
 separate label). **Only listed locales are indexed** (searched and sorted); a
 literal whose language tag is not in `locales` is still **displayed** but not
 matched or sorted on. Display fields are **omitted, never empty**, when a document
-lacks that language, and the per-locale search/sort fields likewise, so declare
-them optional in the engine schema and sort with `missing_values: last`. A
+lacks that language, and the per-locale search fields likewise, so declare them
+optional in the engine schema and sort with `missing_values: last`. A sort field
+is present whenever the field holds any value in a declared locale (the fallback
+above); it is absent only when the field is. A
 deployment that wants to bound the displayed languages narrows them upstream
 (e.g. selecting a language subset in its CONSTRUCT query), since preservation is
 the default.

packages/search-pipeline/test/registry-extraction.integration.test.ts

@@ -160,6 +160,10 @@ describe('registry-sourced extraction: scoped to the dataset’s own graph', ()
         label_sort_nl: 'vaandels van limburg',
         label_search_en: 'banners of limburg',
         label_sort_en: 'banners of limburg',
+        // The `und` locale is declared but untagged labels are absent here, so
+        // its sort key falls back to the first declared locale’s value – a
+        // reader whose language the request does not state still gets an order.
+        label_sort_und: 'vaandels van limburg',
         description_nl: 'Een dataset over vaandels.',
         description_search_nl: 'een dataset over vaandels.',
         publisher: ['https://ex/org/trace'],

packages/search/src/project.ts

@@ -455,6 +455,17 @@ function referenceValues(
  * `sortable`) stay on the declared `locales`, which drive the indexed, stemmed,
  * weighted fanout; a value in an undeclared language is not indexed. Absent
  * languages emit nothing.
+ *
+ * The two companions differ in what an absent language means. A search query
+ * fans out over *every* locale key at once, so a document titled in one
+ * language is found whichever language the reader asks in. A sort names a
+ * **single** key, so a locale key left empty is not “no value in Dutch” – it is
+ * the empty string, which ties with every other document missing that language
+ * and leaves them in relevance order. Each locale’s sort key therefore falls
+ * back to the document’s first value in `locales` order: a collection of
+ * untagged titles sorts by title in a Dutch request rather than not at all.
+ * Ordering across languages is approximate by nature; a total order over the
+ * titles a reader actually sees beats a partial one over the tagged few.
  */
 function applyText(
   document: ProjectedNode,
@@ -476,22 +487,30 @@ function applyText(
   // here it simply indexes nothing.
   if (field.searchable !== undefined || field.sortable === true) {
     const names = physicalFields(field);
-    field.locales.forEach((locale, index) => {
-      const localeValues = values
+    const valuesPerLocale = field.locales.map((locale) =>
+      values
         .filter((value) => value.lang === locale)
-        .map((value) => value.value);
-      if (localeValues.length === 0) {
-        return;
-      }
-      if (field.searchable) {
+        .map((value) => value.value),
+    );
+    // What every locale’s sort key falls back to: the document’s first value
+    // in `locales` order, which is the declaration’s own statement of which
+    // language stands in for the others.
+    const fallbackSortValue = valuesPerLocale.find(
+      (localeValues) => localeValues.length > 0,
+    )?.[0];
+    valuesPerLocale.forEach((localeValues, index) => {
+      if (field.searchable !== undefined && localeValues.length > 0) {
         setString(
           document,
           names.search[index],
           foldedSearchValue(localeValues),
         );
       }
-      if (field.sortable) {
-        setString(document, names.sort[index], fold(localeValues[0]));
+      if (field.sortable === true) {
+        const sortValue = localeValues[0] ?? fallbackSortValue;
+        if (sortValue !== undefined) {
+          setString(document, names.sort[index], fold(sortValue));
+        }
       }
     });
   }

packages/search/test/project.test.ts

@@ -351,6 +351,24 @@ describe('projectDocument', () => {
     expect(document.title_sort_nl).toBeUndefined();
   });
 
+  it('falls back to the document’s first value for a locale it has no title in', () => {
+    const document = projectDocument(
+      {
+        '@id': 'https://ex/d/5b',
+        [dsKey('title')]: { '@language': 'en', '@value': 'Title' },
+      },
+      { name: 'Dataset', class: DATASET, fields },
+    );
+    // A sort names one key, so an empty `title_sort_nl` would tie this document
+    // with every other one lacking a Dutch title – no order at all. It sorts
+    // under its English title instead. Search is unaffected: that query fans
+    // out over every locale key, so the fallback would only duplicate.
+    expect(document.title_sort_nl).toBe('title');
+    expect(document.title_sort_en).toBe('title');
+    expect(document.title_search_nl).toBeUndefined();
+    expect(document.title_search_en).toBe('title');
+  });
+
   it('displays a value whose language is outside locales, but does not index it', () => {
     const document = projectDocument(
       {