From 688f067ebbb5d18c5655c4458f10b552f808a693 Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:26:18 +0530 Subject: [PATCH 1/3] fix(search): scrub stale file extension aggregation on 1.12.10 upgrade (#1A file search 500) Backport of the 1.13 fix (main PR #28555) to the 1.12.10 release line. PR #27080 (backported to 1.12.5) changed the file index mapping from extension:keyword to extension:flattened and renamed the seed aggregation to fileExtension. The SettingsCache startup merge is additive for existing asset types, so any cluster whose DB already held the file config (upgraded from pre-1.12.5) keeps the stale extension aggregation. On OpenSearch, a terms agg against a flat_object field throws illegal_argument_exception, causing a 500 on every file search query. Adds v11210 migration with removeStaleFileExtensionAggregation() targeting only the file assetType aggregation list. Idempotent. --- .../native/1.12.10/mysql/schemaChanges.sql | 3 + .../native/1.12.10/postgres/schemaChanges.sql | 3 + .../migration/mysql/v11210/Migration.java | 21 +++ .../migration/postgres/v11210/Migration.java | 21 +++ .../migration/utils/v11210/MigrationUtil.java | 70 ++++++++++ .../FileExtensionAggregationScrubTest.java | 126 ++++++++++++++++++ 6 files changed, 244 insertions(+) create mode 100644 bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql create mode 100644 bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java diff --git a/bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql new file mode 100644 index 000000000000..c75ad3e0961f --- /dev/null +++ b/bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql @@ -0,0 +1,3 @@ +-- Placeholder for 1.12.10 MySQL schema changes +-- No DDL changes required; this version only carries a data migration +-- (scrub of stale file extension aggregation from SearchSettings). diff --git a/bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql new file mode 100644 index 000000000000..72d36e12c915 --- /dev/null +++ b/bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql @@ -0,0 +1,3 @@ +-- Placeholder for 1.12.10 Postgres schema changes +-- No DDL changes required; this version only carries a data migration +-- (scrub of stale file extension aggregation from SearchSettings). diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java new file mode 100644 index 000000000000..5b400a7658bc --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java @@ -0,0 +1,21 @@ +package org.openmetadata.service.migration.mysql.v11210; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; +import org.openmetadata.service.migration.utils.v11210.MigrationUtil; + +@Slf4j +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + MigrationUtil.removeStaleFileExtensionAggregation(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java new file mode 100644 index 000000000000..2d6af3344ebf --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java @@ -0,0 +1,21 @@ +package org.openmetadata.service.migration.postgres.v11210; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.service.migration.api.MigrationProcessImpl; +import org.openmetadata.service.migration.utils.MigrationFile; +import org.openmetadata.service.migration.utils.v11210.MigrationUtil; + +@Slf4j +public class Migration extends MigrationProcessImpl { + + public Migration(MigrationFile migrationFile) { + super(migrationFile); + } + + @Override + @SneakyThrows + public void runDataMigration() { + MigrationUtil.removeStaleFileExtensionAggregation(); + } +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java new file mode 100644 index 000000000000..73c44cfd0dc4 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java @@ -0,0 +1,70 @@ +package org.openmetadata.service.migration.utils.v11210; + +import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.openmetadata.schema.api.search.Aggregation; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.SearchSettings; +import org.openmetadata.schema.settings.Settings; +import org.openmetadata.service.migration.utils.SearchSettingsMergeUtil; + +@Slf4j +public class MigrationUtil { + private MigrationUtil() {} + + // PR #27080 (backported to 1.12.5) renamed the file aggregation field from "extension" + // (flattened, unsupported by OpenSearch terms agg) to "fileExtension" (keyword). The seed was + // updated but the additive settings merge means clusters upgraded from pre-1.12.5 still carry + // the stale aggregation, causing a 500 on every file search query on OpenSearch. + private static final String STALE_FILE_EXTENSION_AGGREGATION_FIELD = "extension"; + private static final String FILE_ASSET_TYPE = "file"; + + /** + * Removes the stale {@code extension} aggregation from the DB-stored file SearchSettings on + * upgraded clusters. Idempotent; safe to call on every reprocessing pass. + */ + public static void removeStaleFileExtensionAggregation() { + try { + Settings searchSettings = SearchSettingsMergeUtil.getSearchSettingsFromDatabase(); + if (searchSettings == null) { + LOG.warn("Search settings not found in database; skipping stale file extension agg scrub"); + return; + } + SearchSettings currentSettings = SearchSettingsMergeUtil.loadSearchSettings(searchSettings); + if (stripStaleFileExtensionAggregation(currentSettings)) { + SearchSettingsMergeUtil.saveSearchSettings(searchSettings, currentSettings); + LOG.info("Removed stale 'extension' aggregation from file search settings"); + } else { + LOG.info("No stale 'extension' aggregation found in file search settings"); + } + } catch (Exception e) { + LOG.error("Error removing stale file extension aggregation from search settings", e); + } + } + + public static boolean stripStaleFileExtensionAggregation(SearchSettings settings) { + boolean changed = false; + if (settings == null) { + return false; + } + for (AssetTypeConfiguration assetConfig : listOrEmpty(settings.getAssetTypeConfigurations())) { + if (FILE_ASSET_TYPE.equals(assetConfig.getAssetType())) { + changed |= removeStaleAggregation(assetConfig.getAggregations()); + } + } + return changed; + } + + private static boolean removeStaleAggregation(List aggregations) { + boolean removed = false; + if (!nullOrEmpty(aggregations)) { + removed = + aggregations.removeIf( + agg -> STALE_FILE_EXTENSION_AGGREGATION_FIELD.equals(agg.getField())); + } + return removed; + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java new file mode 100644 index 000000000000..838ce5b0207b --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java @@ -0,0 +1,126 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.migration.utils.v11210; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.ArrayList; +import java.util.List; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.api.search.Aggregation; +import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.SearchSettings; + +class FileExtensionAggregationScrubTest { + + private Aggregation agg(String field) { + Aggregation a = new Aggregation(); + a.setField(field); + a.setName(field); + return a; + } + + private AssetTypeConfiguration assetConfig(String assetType, Aggregation... aggregations) { + AssetTypeConfiguration config = new AssetTypeConfiguration(); + config.setAssetType(assetType); + config.setAggregations(new ArrayList<>(List.of(aggregations))); + return config; + } + + @Test + void scrubRemovesStaleExtensionAggregationFromFileConfig() { + SearchSettings settings = new SearchSettings(); + settings.setAssetTypeConfigurations( + new ArrayList<>( + List.of( + assetConfig( + "file", + agg("fileType"), + agg("extension"), + agg("fileExtension"), + agg("service.displayName.keyword"))))); + + boolean changed = MigrationUtil.stripStaleFileExtensionAggregation(settings); + + assertTrue( + changed, "Scrub should report a change when the stale extension aggregation is present"); + List aggs = settings.getAssetTypeConfigurations().getFirst().getAggregations(); + assertEquals(3, aggs.size()); + assertTrue( + aggs.stream().noneMatch(a -> "extension".equals(a.getField())), + "Stale 'extension' aggregation must be removed"); + assertTrue( + aggs.stream().anyMatch(a -> "fileExtension".equals(a.getField())), + "Current 'fileExtension' aggregation must be preserved"); + } + + @Test + void scrubDoesNotTouchOtherAssetTypes() { + SearchSettings settings = new SearchSettings(); + settings.setAssetTypeConfigurations( + new ArrayList<>( + List.of( + assetConfig("file", agg("extension"), agg("fileType")), + assetConfig("table", agg("extension"), agg("columns.name")), + assetConfig("topic", agg("extension"), agg("messageSchema.schemaType"))))); + + boolean changed = MigrationUtil.stripStaleFileExtensionAggregation(settings); + + assertTrue(changed, "Scrub should report change for the file asset type"); + List fileAggs = settings.getAssetTypeConfigurations().get(0).getAggregations(); + assertEquals(1, fileAggs.size(), "Only file aggregations should be modified"); + assertEquals("fileType", fileAggs.getFirst().getField()); + + List tableAggs = settings.getAssetTypeConfigurations().get(1).getAggregations(); + assertEquals(2, tableAggs.size(), "table aggregations must not be touched"); + + List topicAggs = settings.getAssetTypeConfigurations().get(2).getAggregations(); + assertEquals(2, topicAggs.size(), "topic aggregations must not be touched"); + } + + @Test + void scrubIsIdempotentWhenExtensionAlreadyAbsent() { + SearchSettings settings = new SearchSettings(); + settings.setAssetTypeConfigurations( + new ArrayList<>( + List.of( + assetConfig( + "file", + agg("fileType"), + agg("fileExtension"), + agg("directory.displayName.keyword"), + agg("service.displayName.keyword"))))); + + boolean changed = MigrationUtil.stripStaleFileExtensionAggregation(settings); + + assertFalse(changed, "Scrub must not report a change when extension is already absent"); + assertEquals(4, settings.getAssetTypeConfigurations().getFirst().getAggregations().size()); + } + + @Test + void scrubHandlesNullAndEmptyGracefully() { + assertFalse(MigrationUtil.stripStaleFileExtensionAggregation(null)); + + SearchSettings empty = new SearchSettings(); + assertFalse(MigrationUtil.stripStaleFileExtensionAggregation(empty)); + + SearchSettings nullAggs = new SearchSettings(); + AssetTypeConfiguration fileConfig = new AssetTypeConfiguration(); + fileConfig.setAssetType("file"); + fileConfig.setAggregations(null); + nullAggs.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig))); + assertFalse(MigrationUtil.stripStaleFileExtensionAggregation(nullAggs)); + } +} From ed1458db664d18bee142deea151b6645d62116a2 Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Mon, 1 Jun 2026 13:08:01 +0530 Subject: [PATCH 2/3] fix(search): also scrub stale extension from file searchFields in v11210 PR #27080 renamed both the aggregation and searchField from extension to fileExtension. Extended stripStaleFileExtensionSettings to also remove extension from searchFields, preventing multi_match failures on non-empty file queries on OpenSearch. --- .../migration/utils/v11210/MigrationUtil.java | 40 +++++--- .../FileExtensionAggregationScrubTest.java | 97 ++++++++++--------- 2 files changed, 77 insertions(+), 60 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java index 73c44cfd0dc4..e749e3fe3fc4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java @@ -7,6 +7,7 @@ import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.search.Aggregation; import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.FieldBoost; import org.openmetadata.schema.api.search.SearchSettings; import org.openmetadata.schema.settings.Settings; import org.openmetadata.service.migration.utils.SearchSettingsMergeUtil; @@ -15,37 +16,37 @@ public class MigrationUtil { private MigrationUtil() {} - // PR #27080 (backported to 1.12.5) renamed the file aggregation field from "extension" - // (flattened, unsupported by OpenSearch terms agg) to "fileExtension" (keyword). The seed was - // updated but the additive settings merge means clusters upgraded from pre-1.12.5 still carry - // the stale aggregation, causing a 500 on every file search query on OpenSearch. - private static final String STALE_FILE_EXTENSION_AGGREGATION_FIELD = "extension"; + // PR #27080 renamed the file search field and aggregation from "extension" (flattened — + // unsupported for terms agg and multi_match on OpenSearch) to "fileExtension" (keyword). The + // seed was updated but the additive settings merge means clusters upgraded from pre-1.12.5 + // still carry both stale entries, causing a 500 on every file search query on OpenSearch. + private static final String STALE_FILE_EXTENSION_FIELD = "extension"; private static final String FILE_ASSET_TYPE = "file"; /** - * Removes the stale {@code extension} aggregation from the DB-stored file SearchSettings on - * upgraded clusters. Idempotent; safe to call on every reprocessing pass. + * Removes the stale {@code extension} searchField and aggregation from the DB-stored file + * SearchSettings on upgraded clusters. Idempotent; safe to call on every reprocessing pass. */ public static void removeStaleFileExtensionAggregation() { try { Settings searchSettings = SearchSettingsMergeUtil.getSearchSettingsFromDatabase(); if (searchSettings == null) { - LOG.warn("Search settings not found in database; skipping stale file extension agg scrub"); + LOG.warn("Search settings not found in database; skipping stale file extension scrub"); return; } SearchSettings currentSettings = SearchSettingsMergeUtil.loadSearchSettings(searchSettings); - if (stripStaleFileExtensionAggregation(currentSettings)) { + if (stripStaleFileExtensionSettings(currentSettings)) { SearchSettingsMergeUtil.saveSearchSettings(searchSettings, currentSettings); - LOG.info("Removed stale 'extension' aggregation from file search settings"); + LOG.info("Removed stale 'extension' searchField and aggregation from file search settings"); } else { - LOG.info("No stale 'extension' aggregation found in file search settings"); + LOG.info("No stale 'extension' entries found in file search settings"); } } catch (Exception e) { - LOG.error("Error removing stale file extension aggregation from search settings", e); + LOG.error("Error removing stale file extension settings", e); } } - public static boolean stripStaleFileExtensionAggregation(SearchSettings settings) { + public static boolean stripStaleFileExtensionSettings(SearchSettings settings) { boolean changed = false; if (settings == null) { return false; @@ -53,6 +54,7 @@ public static boolean stripStaleFileExtensionAggregation(SearchSettings settings for (AssetTypeConfiguration assetConfig : listOrEmpty(settings.getAssetTypeConfigurations())) { if (FILE_ASSET_TYPE.equals(assetConfig.getAssetType())) { changed |= removeStaleAggregation(assetConfig.getAggregations()); + changed |= removeStaleSearchField(assetConfig.getSearchFields()); } } return changed; @@ -61,9 +63,15 @@ public static boolean stripStaleFileExtensionAggregation(SearchSettings settings private static boolean removeStaleAggregation(List aggregations) { boolean removed = false; if (!nullOrEmpty(aggregations)) { - removed = - aggregations.removeIf( - agg -> STALE_FILE_EXTENSION_AGGREGATION_FIELD.equals(agg.getField())); + removed = aggregations.removeIf(agg -> STALE_FILE_EXTENSION_FIELD.equals(agg.getField())); + } + return removed; + } + + private static boolean removeStaleSearchField(List searchFields) { + boolean removed = false; + if (!nullOrEmpty(searchFields)) { + removed = searchFields.removeIf(field -> STALE_FILE_EXTENSION_FIELD.equals(field.getField())); } return removed; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java index 838ce5b0207b..9a9623ac7626 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/migration/utils/v11210/FileExtensionAggregationScrubTest.java @@ -21,6 +21,7 @@ import org.junit.jupiter.api.Test; import org.openmetadata.schema.api.search.Aggregation; import org.openmetadata.schema.api.search.AssetTypeConfiguration; +import org.openmetadata.schema.api.search.FieldBoost; import org.openmetadata.schema.api.search.SearchSettings; class FileExtensionAggregationScrubTest { @@ -32,6 +33,12 @@ private Aggregation agg(String field) { return a; } + private FieldBoost searchField(String field) { + FieldBoost fb = new FieldBoost(); + fb.setField(field); + return fb; + } + private AssetTypeConfiguration assetConfig(String assetType, Aggregation... aggregations) { AssetTypeConfiguration config = new AssetTypeConfiguration(); config.setAssetType(assetType); @@ -40,30 +47,30 @@ private AssetTypeConfiguration assetConfig(String assetType, Aggregation... aggr } @Test - void scrubRemovesStaleExtensionAggregationFromFileConfig() { - SearchSettings settings = new SearchSettings(); - settings.setAssetTypeConfigurations( + void scrubRemovesStaleExtensionAggregationAndSearchFieldFromFileConfig() { + AssetTypeConfiguration fileConfig = new AssetTypeConfiguration(); + fileConfig.setAssetType("file"); + fileConfig.setAggregations( + new ArrayList<>(List.of(agg("fileType"), agg("extension"), agg("fileExtension")))); + fileConfig.setSearchFields( new ArrayList<>( - List.of( - assetConfig( - "file", - agg("fileType"), - agg("extension"), - agg("fileExtension"), - agg("service.displayName.keyword"))))); + List.of(searchField("name"), searchField("extension"), searchField("fileExtension")))); + + SearchSettings settings = new SearchSettings(); + settings.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig))); - boolean changed = MigrationUtil.stripStaleFileExtensionAggregation(settings); + boolean changed = MigrationUtil.stripStaleFileExtensionSettings(settings); - assertTrue( - changed, "Scrub should report a change when the stale extension aggregation is present"); + assertTrue(changed); List aggs = settings.getAssetTypeConfigurations().getFirst().getAggregations(); - assertEquals(3, aggs.size()); - assertTrue( - aggs.stream().noneMatch(a -> "extension".equals(a.getField())), - "Stale 'extension' aggregation must be removed"); - assertTrue( - aggs.stream().anyMatch(a -> "fileExtension".equals(a.getField())), - "Current 'fileExtension' aggregation must be preserved"); + assertEquals(2, aggs.size()); + assertTrue(aggs.stream().noneMatch(a -> "extension".equals(a.getField()))); + assertTrue(aggs.stream().anyMatch(a -> "fileExtension".equals(a.getField()))); + + List sfs = settings.getAssetTypeConfigurations().getFirst().getSearchFields(); + assertEquals(2, sfs.size()); + assertTrue(sfs.stream().noneMatch(f -> "extension".equals(f.getField()))); + assertTrue(sfs.stream().anyMatch(f -> "fileExtension".equals(f.getField()))); } @Test @@ -76,51 +83,53 @@ void scrubDoesNotTouchOtherAssetTypes() { assetConfig("table", agg("extension"), agg("columns.name")), assetConfig("topic", agg("extension"), agg("messageSchema.schemaType"))))); - boolean changed = MigrationUtil.stripStaleFileExtensionAggregation(settings); + boolean changed = MigrationUtil.stripStaleFileExtensionSettings(settings); - assertTrue(changed, "Scrub should report change for the file asset type"); + assertTrue(changed); List fileAggs = settings.getAssetTypeConfigurations().get(0).getAggregations(); - assertEquals(1, fileAggs.size(), "Only file aggregations should be modified"); + assertEquals(1, fileAggs.size()); assertEquals("fileType", fileAggs.getFirst().getField()); - List tableAggs = settings.getAssetTypeConfigurations().get(1).getAggregations(); - assertEquals(2, tableAggs.size(), "table aggregations must not be touched"); - - List topicAggs = settings.getAssetTypeConfigurations().get(2).getAggregations(); - assertEquals(2, topicAggs.size(), "topic aggregations must not be touched"); + assertEquals(2, settings.getAssetTypeConfigurations().get(1).getAggregations().size()); + assertEquals(2, settings.getAssetTypeConfigurations().get(2).getAggregations().size()); } @Test void scrubIsIdempotentWhenExtensionAlreadyAbsent() { - SearchSettings settings = new SearchSettings(); - settings.setAssetTypeConfigurations( + AssetTypeConfiguration fileConfig = new AssetTypeConfiguration(); + fileConfig.setAssetType("file"); + fileConfig.setAggregations( new ArrayList<>( List.of( - assetConfig( - "file", - agg("fileType"), - agg("fileExtension"), - agg("directory.displayName.keyword"), - agg("service.displayName.keyword"))))); + agg("fileType"), + agg("fileExtension"), + agg("directory.displayName.keyword"), + agg("service.displayName.keyword")))); + fileConfig.setSearchFields( + new ArrayList<>(List.of(searchField("name"), searchField("fileExtension")))); + + SearchSettings settings = new SearchSettings(); + settings.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig))); - boolean changed = MigrationUtil.stripStaleFileExtensionAggregation(settings); + boolean changed = MigrationUtil.stripStaleFileExtensionSettings(settings); - assertFalse(changed, "Scrub must not report a change when extension is already absent"); + assertFalse(changed); assertEquals(4, settings.getAssetTypeConfigurations().getFirst().getAggregations().size()); + assertEquals(2, settings.getAssetTypeConfigurations().getFirst().getSearchFields().size()); } @Test void scrubHandlesNullAndEmptyGracefully() { - assertFalse(MigrationUtil.stripStaleFileExtensionAggregation(null)); + assertFalse(MigrationUtil.stripStaleFileExtensionSettings(null)); - SearchSettings empty = new SearchSettings(); - assertFalse(MigrationUtil.stripStaleFileExtensionAggregation(empty)); + assertFalse(MigrationUtil.stripStaleFileExtensionSettings(new SearchSettings())); - SearchSettings nullAggs = new SearchSettings(); + SearchSettings nullFields = new SearchSettings(); AssetTypeConfiguration fileConfig = new AssetTypeConfiguration(); fileConfig.setAssetType("file"); fileConfig.setAggregations(null); - nullAggs.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig))); - assertFalse(MigrationUtil.stripStaleFileExtensionAggregation(nullAggs)); + fileConfig.setSearchFields(null); + nullFields.setAssetTypeConfigurations(new ArrayList<>(List.of(fileConfig))); + assertFalse(MigrationUtil.stripStaleFileExtensionSettings(nullFields)); } } From d5a6350f61c0425221600807b23f111671a123d4 Mon Sep 17 00:00:00 2001 From: sonika-shah <58761340+sonika-shah@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:41:27 +0530 Subject: [PATCH 3/3] =?UTF-8?q?fix(search):=20sync=20v11210=20backport=20w?= =?UTF-8?q?ith=20main=20=E2=80=94=20add=20flattened-children=20scrub?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backport branch was missing removeFlattenedChildrenSearchSettings() which was already in main's v11210 via conflict resolution with the immense-term children fix. Both scrubs now run in v11210: flattened-children references and stale file extension aggregation/searchField. --- .../native/1.12.10/mysql/schemaChanges.sql | 2 +- .../native/1.12.10/postgres/schemaChanges.sql | 2 +- .../migration/mysql/v11210/Migration.java | 13 +++ .../migration/postgres/v11210/Migration.java | 13 +++ .../migration/utils/v11210/MigrationUtil.java | 97 ++++++++++++++++++- 5 files changed, 121 insertions(+), 6 deletions(-) diff --git a/bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql index c75ad3e0961f..75ef2c8e2623 100644 --- a/bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.12.10/mysql/schemaChanges.sql @@ -1,3 +1,3 @@ -- Placeholder for 1.12.10 MySQL schema changes -- No DDL changes required; this version only carries a data migration --- (scrub of stale file extension aggregation from SearchSettings). +-- (scrub of stale flattened-children references and file extension aggregation from SearchSettings). diff --git a/bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql index 72d36e12c915..b3f7fad5b5f4 100644 --- a/bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.12.10/postgres/schemaChanges.sql @@ -1,3 +1,3 @@ -- Placeholder for 1.12.10 Postgres schema changes -- No DDL changes required; this version only carries a data migration --- (scrub of stale file extension aggregation from SearchSettings). +-- (scrub of stale flattened-children references and file extension aggregation from SearchSettings). diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java index 5b400a7658bc..ebf33dd5337b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v11210/Migration.java @@ -1,3 +1,15 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.openmetadata.service.migration.mysql.v11210; import lombok.SneakyThrows; @@ -16,6 +28,7 @@ public Migration(MigrationFile migrationFile) { @Override @SneakyThrows public void runDataMigration() { + MigrationUtil.removeFlattenedChildrenSearchSettings(); MigrationUtil.removeStaleFileExtensionAggregation(); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java index 2d6af3344ebf..aa0eedd30b74 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v11210/Migration.java @@ -1,3 +1,15 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.openmetadata.service.migration.postgres.v11210; import lombok.SneakyThrows; @@ -16,6 +28,7 @@ public Migration(MigrationFile migrationFile) { @Override @SneakyThrows public void runDataMigration() { + MigrationUtil.removeFlattenedChildrenSearchSettings(); MigrationUtil.removeStaleFileExtensionAggregation(); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java index e749e3fe3fc4..c03a43d73f08 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v11210/MigrationUtil.java @@ -1,11 +1,25 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.openmetadata.service.migration.utils.v11210; import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; import java.util.List; +import java.util.Set; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.api.search.Aggregation; +import org.openmetadata.schema.api.search.AllowedSearchFields; import org.openmetadata.schema.api.search.AssetTypeConfiguration; import org.openmetadata.schema.api.search.FieldBoost; import org.openmetadata.schema.api.search.SearchSettings; @@ -16,6 +30,85 @@ public class MigrationUtil { private MigrationUtil() {} + // The recursive column/schema children subtree was remapped from flattened to + // object/enabled:false. Its indexed leaves no longer exist so references to + // columns.children.name, dataModel.columns.children.name, and the schemaFields + // variants in highlightFields, searchFields, and allowedFields are now dead weight. + private static final Set STALE_FLATTENED_CHILDREN_FIELDS = + Set.of( + "columns.children.name", + "columns.children.description", + "dataModel.columns.children.name", + "messageSchema.schemaFields.children.name", + "requestSchema.schemaFields.children.name", + "responseSchema.schemaFields.children.name"); + + public static void removeFlattenedChildrenSearchSettings() { + try { + Settings searchSettings = SearchSettingsMergeUtil.getSearchSettingsFromDatabase(); + if (searchSettings == null) { + LOG.warn("Search settings not found in database; skipping flattened-children scrub"); + } else { + SearchSettings currentSettings = SearchSettingsMergeUtil.loadSearchSettings(searchSettings); + if (stripFlattenedChildrenReferences(currentSettings)) { + SearchSettingsMergeUtil.saveSearchSettings(searchSettings, currentSettings); + LOG.info("Removed stale flattened children references from search settings"); + } else { + LOG.info("No stale flattened children references found in search settings"); + } + } + } catch (Exception e) { + LOG.error("Error removing stale flattened children references from search settings", e); + } + } + + public static boolean stripFlattenedChildrenReferences(SearchSettings settings) { + boolean changed = false; + if (settings != null) { + if (settings.getGlobalSettings() != null) { + changed = removeStaleHighlightFields(settings.getGlobalSettings().getHighlightFields()); + } + for (AssetTypeConfiguration assetConfig : + listOrEmpty(settings.getAssetTypeConfigurations())) { + changed |= removeStaleHighlightFields(assetConfig.getHighlightFields()); + changed |= removeStaleSearchFields(assetConfig.getSearchFields()); + } + changed |= removeStaleAllowedFields(settings.getAllowedFields()); + } + return changed; + } + + private static boolean removeStaleHighlightFields(List highlightFields) { + boolean removed = false; + if (!nullOrEmpty(highlightFields)) { + removed = highlightFields.removeIf(STALE_FLATTENED_CHILDREN_FIELDS::contains); + } + return removed; + } + + private static boolean removeStaleSearchFields(List searchFields) { + boolean removed = false; + if (!nullOrEmpty(searchFields)) { + removed = + searchFields.removeIf( + field -> STALE_FLATTENED_CHILDREN_FIELDS.contains(field.getField())); + } + return removed; + } + + private static boolean removeStaleAllowedFields(List allowedFields) { + boolean removed = false; + for (AllowedSearchFields allowedField : listOrEmpty(allowedFields)) { + if (!nullOrEmpty(allowedField.getFields())) { + removed |= + allowedField + .getFields() + .removeIf(field -> STALE_FLATTENED_CHILDREN_FIELDS.contains(field.getName())); + } + } + return removed; + } + // PR #27080 renamed the file search field and aggregation from "extension" (flattened — // unsupported for terms agg and multi_match on OpenSearch) to "fileExtension" (keyword). The // seed was updated but the additive settings merge means clusters upgraded from pre-1.12.5 @@ -23,10 +116,6 @@ private MigrationUtil() {} private static final String STALE_FILE_EXTENSION_FIELD = "extension"; private static final String FILE_ASSET_TYPE = "file"; - /** - * Removes the stale {@code extension} searchField and aggregation from the DB-stored file - * SearchSettings on upgraded clusters. Idempotent; safe to call on every reprocessing pass. - */ public static void removeStaleFileExtensionAggregation() { try { Settings searchSettings = SearchSettingsMergeUtil.getSearchSettingsFromDatabase();