fix(search): scrub stale file extension aggregation on 1.12.10 upgrade (#1A file search 500)

bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql

@@ -1,3 +1,3 @@
 -- Placeholder for 1.12.10 MySQL schema changes
 -- No DDL changes required; this version only carries a data migration
--- (scrub of stale flattened-children references from SearchSettings).
+-- (scrub of stale flattened-children references and file extension aggregation from SearchSettings).

bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql

@@ -1,3 +1,3 @@
 -- Placeholder for 1.12.10 Postgres schema changes
 -- No DDL changes required; this version only carries a data migration
--- (scrub of stale flattened-children references from SearchSettings).
+-- (scrub of stale flattened-children references and file extension aggregation from SearchSettings).

openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java

@@ -29,5 +29,6 @@ public Migration(MigrationFile migrationFile) {
   @SneakyThrows
   public void runDataMigration() {
     MigrationUtil.removeFlattenedChildrenSearchSettings();
+    MigrationUtil.removeStaleFileExtensionAggregation();
   }
 }

openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java

@@ -29,5 +29,6 @@ public Migration(MigrationFile migrationFile) {
   @SneakyThrows
   public void runDataMigration() {
     MigrationUtil.removeFlattenedChildrenSearchSettings();
+    MigrationUtil.removeStaleFileExtensionAggregation();
   }
 }

openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java

@@ -18,6 +18,7 @@
 import java.util.List;
 import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.schema.api.search.Aggregation;
 import org.openmetadata.schema.api.search.AllowedSearchFields;
 import org.openmetadata.schema.api.search.AssetTypeConfiguration;
 import org.openmetadata.schema.api.search.FieldBoost;
@@ -33,8 +34,6 @@ private MigrationUtil() {}
   // object/enabled:false. Its indexed leaves no longer exist so references to
   // columns.children.name, dataModel.columns.children.name, and the schemaFields
   // variants in highlightFields, searchFields, and allowedFields are now dead weight.
-  // Upgraded clusters keep the stale entries because the additive settings merge
-  // preserves stored asset config wholesale; this scrub removes exactly those entries.
   private static final Set<String> STALE_FLATTENED_CHILDREN_FIELDS =
       Set.of(
           "columns.children.name",
@@ -44,12 +43,6 @@ private MigrationUtil() {}
           "requestSchema.schemaFields.children.name",
           "responseSchema.schemaFields.children.name");
 
-  /**
-   * Removes every reference to the now non-indexed {@code *.children} fields from the DB-stored
-   * SearchSettings on clusters upgrading to 1.12.10. The recursive column/schema {@code children}
-   * subtree is mapped {@code object} with {@code "enabled": false} — stored for display, not
-   * indexed — so its leaves no longer resolve. Idempotent; safe to call on every reprocessing pass.
-   */
   public static void removeFlattenedChildrenSearchSettings() {
     try {
       Settings searchSettings = SearchSettingsMergeUtil.getSearchSettingsFromDatabase();
@@ -115,4 +108,60 @@ private static boolean removeStaleAllowedFields(List<AllowedSearchFields> allowe
     }
     return removed;
   }
+
+  // PR #27080 renamed the file search field and aggregation from "extension" (flattened —
+  // unsupported for terms agg and multi_match on OpenSearch) to "fileExtension" (keyword). The
+  // seed was updated but the additive settings merge means clusters upgraded from pre-1.12.5
+  // still carry both stale entries, causing a 500 on every file search query on OpenSearch.
+  private static final String STALE_FILE_EXTENSION_FIELD = "extension";
+  private static final String FILE_ASSET_TYPE = "file";
+
+  public static void removeStaleFileExtensionAggregation() {
+    try {
+      Settings searchSettings = SearchSettingsMergeUtil.getSearchSettingsFromDatabase();
+      if (searchSettings == null) {
+        LOG.warn("Search settings not found in database; skipping stale file extension scrub");
+        return;
+      }
+      SearchSettings currentSettings = SearchSettingsMergeUtil.loadSearchSettings(searchSettings);
+      if (stripStaleFileExtensionSettings(currentSettings)) {
+        SearchSettingsMergeUtil.saveSearchSettings(searchSettings, currentSettings);
+        LOG.info("Removed stale 'extension' searchField and aggregation from file search settings");
+      } else {
+        LOG.info("No stale 'extension' entries found in file search settings");
+      }
+    } catch (Exception e) {
+      LOG.error("Error removing stale file extension settings", e);
+    }
+  }
+
+  public static boolean stripStaleFileExtensionSettings(SearchSettings settings) {
+    boolean changed = false;
+    if (settings == null) {
+      return false;
+    }
+    for (AssetTypeConfiguration assetConfig : listOrEmpty(settings.getAssetTypeConfigurations())) {
+      if (FILE_ASSET_TYPE.equals(assetConfig.getAssetType())) {
+        changed |= removeStaleAggregation(assetConfig.getAggregations());
+        changed |= removeStaleSearchField(assetConfig.getSearchFields());
+      }
+    }
+    return changed;
+  }
+
+  private static boolean removeStaleAggregation(List<Aggregation> aggregations) {
+    boolean removed = false;
+    if (!nullOrEmpty(aggregations)) {
+      removed = aggregations.removeIf(agg -> STALE_FILE_EXTENSION_FIELD.equals(agg.getField()));
+    }
+    return removed;
+  }
+
+  private static boolean removeStaleSearchField(List<FieldBoost> searchFields) {
+    boolean removed = false;
+    if (!nullOrEmpty(searchFields)) {
+      removed = searchFields.removeIf(field -> STALE_FILE_EXTENSION_FIELD.equals(field.getField()));
+    }
+    return removed;
+  }
 }

openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java

@@ -0,0 +1,135 @@
+/*
+ *  Copyright 2024 Collate.
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+package org.openmetadata.service.migration.utils.v11210;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.junit.jupiter.api.Test;
+import org.openmetadata.schema.api.search.Aggregation;
+import org.openmetadata.schema.api.search.AssetTypeConfiguration;
+import org.openmetadata.schema.api.search.FieldBoost;
+import org.openmetadata.schema.api.search.SearchSettings;
+
+class FileExtensionAggregationScrubTest {
+
+  private Aggregation agg(String field) {
+    Aggregation a = new Aggregation();
+    a.setField(field);
+    a.setName(field);
+    return a;
+  }
+
+  private FieldBoost searchField(String field) {
+    FieldBoost fb = new FieldBoost();
+    fb.setField(field);
+    return fb;
+  }
+
+  private AssetTypeConfiguration assetConfig(String assetType, Aggregation... aggregations) {
+    AssetTypeConfiguration config = new AssetTypeConfiguration();
+    config.setAssetType(assetType);
+    config.setAggregations(new ArrayList<>(List.of(aggregations)));
+    return config;
+  }
+
+  @Test
+  void scrubRemovesStaleExtensionAggregationAndSearchFieldFromFileConfig() {
+    AssetTypeConfiguration fileConfig = new AssetTypeConfiguration();
+    fileConfig.setAssetType("file");
+    fileConfig.setAggregations(
+        new ArrayList<>(List.of(agg("fileType"), agg("extension"), agg("fileExtension"))));
+    fileConfig.setSearchFields(
+        new ArrayList<>(
+            List.of(searchField("name"), searchField("extension"), searchField("fileExtension"))));
+
+    SearchSettings settings = new SearchSettings();
+    settings.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig)));
+
+    boolean changed = MigrationUtil.stripStaleFileExtensionSettings(settings);
+
+    assertTrue(changed);
+    List<Aggregation> aggs = settings.getAssetTypeConfigurations().getFirst().getAggregations();
+    assertEquals(2, aggs.size());
+    assertTrue(aggs.stream().noneMatch(a -> "extension".equals(a.getField())));
+    assertTrue(aggs.stream().anyMatch(a -> "fileExtension".equals(a.getField())));
+
+    List<FieldBoost> sfs = settings.getAssetTypeConfigurations().getFirst().getSearchFields();
+    assertEquals(2, sfs.size());
+    assertTrue(sfs.stream().noneMatch(f -> "extension".equals(f.getField())));
+    assertTrue(sfs.stream().anyMatch(f -> "fileExtension".equals(f.getField())));
+  }
+
+  @Test
+  void scrubDoesNotTouchOtherAssetTypes() {
+    SearchSettings settings = new SearchSettings();
+    settings.setAssetTypeConfigurations(
+        new ArrayList<>(
+            List.of(
+                assetConfig("file", agg("extension"), agg("fileType")),
+                assetConfig("table", agg("extension"), agg("columns.name")),
+                assetConfig("topic", agg("extension"), agg("messageSchema.schemaType")))));
+
+    boolean changed = MigrationUtil.stripStaleFileExtensionSettings(settings);
+
+    assertTrue(changed);
+    List<Aggregation> fileAggs = settings.getAssetTypeConfigurations().get(0).getAggregations();
+    assertEquals(1, fileAggs.size());
+    assertEquals("fileType", fileAggs.getFirst().getField());
+
+    assertEquals(2, settings.getAssetTypeConfigurations().get(1).getAggregations().size());
+    assertEquals(2, settings.getAssetTypeConfigurations().get(2).getAggregations().size());
+  }
+
+  @Test
+  void scrubIsIdempotentWhenExtensionAlreadyAbsent() {
+    AssetTypeConfiguration fileConfig = new AssetTypeConfiguration();
+    fileConfig.setAssetType("file");
+    fileConfig.setAggregations(
+        new ArrayList<>(
+            List.of(
+                agg("fileType"),
+                agg("fileExtension"),
+                agg("directory.displayName.keyword"),
+                agg("service.displayName.keyword"))));
+    fileConfig.setSearchFields(
+        new ArrayList<>(List.of(searchField("name"), searchField("fileExtension"))));
+
+    SearchSettings settings = new SearchSettings();
+    settings.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig)));
+
+    boolean changed = MigrationUtil.stripStaleFileExtensionSettings(settings);
+
+    assertFalse(changed);
+    assertEquals(4, settings.getAssetTypeConfigurations().getFirst().getAggregations().size());
+    assertEquals(2, settings.getAssetTypeConfigurations().getFirst().getSearchFields().size());
+  }
+
+  @Test
+  void scrubHandlesNullAndEmptyGracefully() {
+    assertFalse(MigrationUtil.stripStaleFileExtensionSettings(null));
+
+    assertFalse(MigrationUtil.stripStaleFileExtensionSettings(new SearchSettings()));
+
+    SearchSettings nullFields = new SearchSettings();
+    AssetTypeConfiguration fileConfig = new AssetTypeConfiguration();
+    fileConfig.setAssetType("file");
+    fileConfig.setAggregations(null);
+    fileConfig.setSearchFields(null);
+    nullFields.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig)));
+    assertFalse(MigrationUtil.stripStaleFileExtensionSettings(nullFields));
+  }
+}