feat(site-search): allow overriding the index analyzer via SITE_SEARCH_ANALYZER

dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/sitesearch/ESSiteSearchAPI.java

@@ -33,9 +33,7 @@
 import com.dotmarketing.util.UUIDGenerator;
 import com.dotmarketing.util.UtilMethods;
 import com.google.common.annotations.VisibleForTesting;
-import java.io.File;
 import java.io.IOException;
-import java.net.URL;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.time.Duration;
@@ -375,12 +373,8 @@ public synchronized boolean createSiteSearchIndex(String indexName, String alias
             return false;
 
         indexName=indexName.toLowerCase();
-        ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
-        URL url = classLoader.getResource("es-sitesearch-settings.json");
-        // read settings and mappings
-        String settings = new String(com.liferay.util.FileUtil.getBytes(new File(url.getPath())));
-        url = classLoader.getResource("es-sitesearch-mapping.json");
-        String mapping = new String(com.liferay.util.FileUtil.getBytes(new File(url.getPath())));
+        String settings = SiteSearchIndexResources.settings("es-sitesearch-settings.json");
+        String mapping = SiteSearchIndexResources.mapping("es-sitesearch-mapping.json");
 
 
         //create index

dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/sitesearch/OSSiteSearchAPI.java

@@ -48,7 +48,6 @@
 import io.vavr.control.Try;
 import java.io.IOException;
 import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
 import java.text.ParseException;
 import java.text.SimpleDateFormat;
 import java.time.Duration;
@@ -503,7 +502,6 @@ public synchronized boolean createSiteSearchIndex(String indexName, final String
         }
 
         indexName = indexName.toLowerCase();
-        final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
         // OpenSearch-format resources, kept separate from their es-*.json counterparts so the OS
         // index lifecycle never depends on an ES-named file. Settings: the legacy
         // es-sitesearch-settings.json uses ES-only token filter syntax (e.g. edgeNGram / side) that
@@ -512,10 +510,8 @@ public synchronized boolean createSiteSearchIndex(String indexName, final String
         // The mapping is functionally identical to es-sitesearch-mapping.json today, but owning a
         // dedicated os-sitesearch-mapping.json decouples the two vendors — a future ES mapping
         // change cannot silently alter OS behaviour.
-        // Read via getResourceAsStream so the index lifecycle works when these resources are packaged
-        // inside a JAR (new File(url.getPath()) only works for filesystem URLs and NPEs if missing).
-        final String settings = readResource(classLoader, "os-sitesearch-settings.json");
-        final String mapping = readResource(classLoader, "os-sitesearch-mapping.json");
+        final String settings = SiteSearchIndexResources.settings("os-sitesearch-settings.json");
+        final String mapping = SiteSearchIndexResources.mapping("os-sitesearch-mapping.json");
 
         try {
             // Create the .os-tagged physical index (osTagged), matching physicalName()/putMapping()
@@ -544,25 +540,6 @@ public synchronized boolean createSiteSearchIndex(String indexName, final String
      * aggregations such as {@code mimeType}). Forwarding to the same untagged physical name used by
      * {@code createIndex}/search/put keeps the mapping on the index that is actually hit.</p>
      */
-    /**
-     * Reads a UTF-8 classpath resource fully into a String via {@code getResourceAsStream}, so it
-     * resolves whether the resource sits on the filesystem or inside a packaged JAR. Throws a clear
-     * {@link DotSearchException} when the resource is absent rather than NPE-ing on a null URL.
-     */
-    private static String readResource(final ClassLoader classLoader, final String resource)
-            throws DotSearchException {
-        try (final InputStream in = classLoader.getResourceAsStream(resource)) {
-            if (in == null) {
-                throw new DotSearchException(
-                        "Required OpenSearch site search resource not found on the classpath: " + resource);
-            }
-            return new String(in.readAllBytes(), StandardCharsets.UTF_8);
-        } catch (final IOException e) {
-            throw new DotSearchException(
-                    "Error reading OpenSearch site search resource " + resource + ": " + e.getMessage(), e);
-        }
-    }
-
     private void putMapping(final String indexName, final String mapping) throws DotSearchException {
         final String endpoint = "/" + physicalName(indexName) + "/_mapping";
         try (final Response response = clientProvider.getClient().generic()

dotCMS/src/enterprise/java/com/dotcms/enterprise/publishing/sitesearch/SiteSearchIndexResources.java

@@ -0,0 +1,116 @@
+/*
+*
+* Copyright (c) 2025 dotCMS LLC
+* Use of this software is governed by the Business Source License included
+* in the LICENSE file found at in the root directory of software.
+* SPDX-License-Identifier: BUSL-1.1
+*
+*/
+
+package com.dotcms.enterprise.publishing.sitesearch;
+
+import com.dotcms.content.index.domain.DotSearchException;
+import com.dotmarketing.util.Config;
+import com.dotmarketing.util.Logger;
+import com.dotmarketing.util.UtilMethods;
+import com.dotmarketing.util.json.JSONException;
+import com.dotmarketing.util.json.JSONObject;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+/**
+ * Loads the bundled site-search index settings/mapping JSON and applies the optional analyzer
+ * override.
+ *
+ * <p>The {@value #ANALYZER_PROPERTY} {@link Config} property (env:
+ * {@code DOT_SITE_SEARCH_ANALYZER}) names any analyzer known to the search cluster — a built-in
+ * language analyzer ({@code cjk}, {@code arabic}, {@code thai}, ...) or a plugin-provided one
+ * ({@code kuromoji}, {@code nori}, {@code smartcn} — the plugin must be installed on every node).
+ * When set, it is applied to every text field of the site-search mapping (replacing the
+ * English-stemming {@code standard_content} on {@code content}, and the plain {@code standard} on
+ * {@code content_raw}/{@code title}/{@code description}/{@code author}); unset keeps the bundled
+ * defaults untouched. The override takes effect on index creation, so existing indices
+ * need a rebuild to pick it up. An analyzer name unknown to the cluster fails index creation
+ * loudly with the engine's error.
+ *
+ * <p>Operational notes: during ES→OS migration dual-write phases the analyzer must be resolvable
+ * by <b>every</b> active engine — a shadow-engine mapping rejection is only logged by the phase
+ * router, leaving the shadow index on dynamic mapping until reads flip. And when changing the
+ * analyzer, run a fresh site-search job rather than "clear index": clearing deletes and recreates
+ * the active index in place, so a rejected mapping would leave the still-default index
+ * mappingless.</p></p>
+ */
+final class SiteSearchIndexResources {
+
+    static final String ANALYZER_PROPERTY = "SITE_SEARCH_ANALYZER";
+
+    // content_raw is not written by the publishers' mapping-visible DTO fields but IS added to every
+    // text/* document (see ESSiteSearchAPI/OSSiteSearchAPI "content_raw"); it must be covered here or
+    // query_string(default_field:*) matches it with default analysis, reintroducing CJK unigram
+    // false positives.
+    private static final List<String> TEXT_FIELDS =
+            List.of("content", "content_raw", "title", "description", "author");
+
+    private SiteSearchIndexResources() {
+    }
+
+    /** The bundled index settings, verbatim. */
+    static String settings(final String resource) {
+        return readResource(resource);
+    }
+
+    /** The bundled index mapping, with the {@value #ANALYZER_PROPERTY} override applied when set. */
+    static String mapping(final String resource) {
+        final String mapping = readResource(resource);
+        final String configured = Config.getStringProperty(ANALYZER_PROPERTY, null);
+        if (!UtilMethods.isSet(configured)) {
+            return mapping;
+        }
+        final String analyzer = configured.trim();
+        try {
+            final JSONObject root = new JSONObject(mapping);
+            final JSONObject properties = root.getJSONObject("properties");
+            for (final String field : TEXT_FIELDS) {
+                properties.getJSONObject(field).put("analyzer", analyzer);
+            }
+            // the ngram subfield still indexes with the edge-ngram analyzer, but must search with
+            // the same analyzer as the main content field to tokenize queries consistently
+            properties.getJSONObject("content").getJSONObject("fields").getJSONObject("ngram")
+                    .put("search_analyzer", analyzer);
+            final String result = root.toString();
+            if (result == null) {
+                // this JSONObject fork returns null instead of throwing on serialization failure
+                throw new DotSearchException("Error serializing site search mapping " + resource
+                        + " after applying " + ANALYZER_PROPERTY + "=" + analyzer);
+            }
+            Logger.info(SiteSearchIndexResources.class,
+                    ANALYZER_PROPERTY + "=" + analyzer + " applied to site search mapping " + resource);
+            return result;
+        } catch (final JSONException e) {
+            throw new DotSearchException("Error applying " + ANALYZER_PROPERTY + "=" + analyzer
+                    + " to site search mapping " + resource + ": " + e.getMessage(), e);
+        }
+    }
+
+    /**
+     * Reads a UTF-8 classpath resource fully into a String via {@code getResourceAsStream}, so it
+     * resolves whether the resource sits on the filesystem or inside a packaged JAR. Throws a clear
+     * {@link DotSearchException} when the resource is absent rather than NPE-ing on a null URL.
+     */
+    private static String readResource(final String resource) {
+        try (final InputStream in =
+                Thread.currentThread().getContextClassLoader().getResourceAsStream(resource)) {
+            if (in == null) {
+                throw new DotSearchException(
+                        "Site search index resource not found on the classpath: " + resource);
+            }
+            return new String(in.readAllBytes(), StandardCharsets.UTF_8);
+        } catch (final IOException e) {
+            throw new DotSearchException(
+                    "Error reading site search index resource " + resource + ": " + e.getMessage(), e);
+        }
+    }
+}

dotCMS/src/main/resources/es-sitesearch-mapping.json

@@ -18,6 +18,9 @@
           }
         }
       },
+      "content_raw": {
+        "type": "text"
+      },
       "host": {
         "type": "keyword",
         "doc_values" : true

dotCMS/src/main/resources/os-sitesearch-mapping.json

@@ -18,6 +18,9 @@
           }
         }
       },
+      "content_raw": {
+        "type": "text"
+      },
       "host": {
         "type": "keyword",
         "doc_values" : true

dotCMS/src/test/java/com/dotcms/enterprise/publishing/sitesearch/SiteSearchIndexResourcesTest.java

@@ -0,0 +1,86 @@
+package com.dotcms.enterprise.publishing.sitesearch;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
+
+import com.dotcms.UnitTestBase;
+import com.dotcms.content.index.domain.DotSearchException;
+import com.dotmarketing.util.Config;
+import com.dotmarketing.util.json.JSONObject;
+import org.junit.After;
+import org.junit.Test;
+
+/**
+ * Unit tests for {@link SiteSearchIndexResources} — the SITE_SEARCH_ANALYZER override applied to
+ * the bundled site-search mapping, and the default (untouched) behavior when it is unset.
+ */
+public class SiteSearchIndexResourcesTest extends UnitTestBase {
+
+    @After
+    public void cleanUp() {
+        Config.setProperty(SiteSearchIndexResources.ANALYZER_PROPERTY, null);
+    }
+
+    /** Unset property: the bundled mapping and settings come back byte-for-byte. */
+    @Test
+    public void default_mapping_and_settings_are_untouched() throws Exception {
+        final String settings = SiteSearchIndexResources.settings("es-sitesearch-settings.json");
+        assertTrue("bundled settings must declare the default analyzer",
+                settings.contains("standard_content"));
+
+        assertEquals("unset property must return the bundled resource verbatim",
+                readBundledResource("es-sitesearch-mapping.json"),
+                SiteSearchIndexResources.mapping("es-sitesearch-mapping.json"));
+    }
+
+    /** Empty/whitespace values behave like unset — the bundled mapping comes back verbatim. */
+    @Test
+    public void blank_analyzer_values_fall_back_to_default() throws Exception {
+        final String bundled = readBundledResource("es-sitesearch-mapping.json");
+        for (final String blank : new String[]{"", "   "}) {
+            Config.setProperty(SiteSearchIndexResources.ANALYZER_PROPERTY, blank);
+            assertEquals("'" + blank + "' must behave like unset",
+                    bundled, SiteSearchIndexResources.mapping("es-sitesearch-mapping.json"));
+        }
+    }
+
+    private static String readBundledResource(final String resource) throws Exception {
+        try (final java.io.InputStream in = Thread.currentThread().getContextClassLoader()
+                .getResourceAsStream(resource)) {
+            return new String(in.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8);
+        }
+    }
+
+    /**
+     * SITE_SEARCH_ANALYZER set: every text field gets the analyzer, and the ngram subfield's
+     * search_analyzer follows it so queries tokenize consistently. Non-text fields stay untouched.
+     */
+    @Test
+    public void analyzer_override_is_applied_to_text_fields() {
+        // padded value also exercises the trim (env files commonly carry stray whitespace)
+        Config.setProperty(SiteSearchIndexResources.ANALYZER_PROPERTY, " cjk ");
+
+        final JSONObject properties = new JSONObject(
+                SiteSearchIndexResources.mapping("os-sitesearch-mapping.json"))
+                .getJSONObject("properties");
+
+        for (final String field : new String[]{"content", "content_raw", "title", "description", "author"}) {
+            assertEquals("cjk", properties.getJSONObject(field).getString("analyzer"));
+        }
+        final JSONObject ngram = properties.getJSONObject("content").getJSONObject("fields")
+                .getJSONObject("ngram");
+        assertEquals("cjk", ngram.getString("search_analyzer"));
+        assertEquals("edge-ngram index analyzer must be preserved",
+                "partial_content", ngram.getString("analyzer"));
+        assertEquals("non-text fields must be untouched",
+                "keyword", properties.getJSONObject("host").getString("type"));
+    }
+
+    /** A missing bundled resource fails loudly instead of NPE-ing. */
+    @Test
+    public void missing_resource_fails_loudly() {
+        assertThrows(DotSearchException.class,
+                () -> SiteSearchIndexResources.mapping("no-such-resource-on-classpath.json"));
+    }
+}