provided Helm configuration for configuring indexedFields for namespaces

* consolidated config keys to be more intuitive

Signed-off-by: Thomas Jäckle <thomas.jaeckle@beyonnex.io>

deployment/helm/ditto/Chart.yaml

@@ -16,7 +16,7 @@ description: |
   A digital twin is a virtual, cloud based, representation of his real world counterpart
   (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc).
 type: application
-version: 3.4.4  # chart version is effectively set by release-job
+version: 3.4.5  # chart version is effectively set by release-job
 appVersion: 3.4.4
 keywords:
   - iot-chart

deployment/helm/ditto/templates/thingssearch-deployment.yaml

@@ -136,6 +136,15 @@ spec:
                 {{- if .Values.global.logging.customConfigFile.enabled }}
                 -Dlogback.configurationFile=/opt/ditto/{{ .Values.global.logging.customConfigFile.fileName }}
                 {{- end }}
+                {{- if .Values.thingsSearch.config.indexedFieldsLimiting.enabled }}
+                -Dditto.extensions.caching-signal-enrichment-facade-provider=org.eclipse.ditto.thingsearch.service.persistence.write.streaming.SearchIndexingSignalEnrichmentFacadeProvider
+                {{- range $index, $value := .Values.thingsSearch.config.indexedFieldsLimiting.items }}
+                "{{ printf "%s%d%s=%s" "-Dditto.search.namespace-indexed-fields." $index ".namespace-pattern" $value.namespacePattern }}"
+                {{- range $fieldIndex, $indexedField := $value.indexedFields }}
+                "{{ printf "%s%d%s%d=%s" "-Dditto.search.namespace-indexed-fields." $index ".indexed-fields." $fieldIndex $indexedField }}"
+                {{- end }}
+                {{- end }}
+                {{- end }}
                 {{- range $key, $value := .Values.thingsSearch.config.operatorMetrics.customMetrics }}
                 "{{ printf "%s%s%s=%t" "-Dditto.search.operator-metrics.custom-metrics." $key ".enabled" $value.enabled }}"
                 "{{ printf "%s%s%s=%s" "-Dditto.search.operator-metrics.custom-metrics." $key ".scrape-interval" $value.scrapeInterval }}"

deployment/helm/ditto/values.yaml

@@ -1077,6 +1077,20 @@ thingsSearch:
           throughput: 100
           # period the throttle period
           period: 30s
+    # indexedFieldsLimiting by default, Ditto indexed all fields of things in its search.
+    #  However, this behavior can be customized, providing configuration to only index certain fields for specified namespaces.
+    indexedFieldsLimiting:
+      # enabled whether field index limiting should be enabled or not
+      enabled: false
+      # items contains the list of per-namespace configuration of which fields to include into to the search index
+      items:
+      #  - # namespacePattern holds the namespace for which the single limiting configuration entry should apply.
+      #    #  Wildcards `*` (Matching any number of any character) and `?` (Matches any single character) are supported.
+      #    namespacePattern: "org.eclipse.*"
+      #    # indexedFields holds a list of fields that will be explicitly included in the search index
+      #    indexedFields:
+      #      - "attributes"
+      #      - "features/included"
     # operatorMetrics contains configuration for operator defined custom metrics, using a search "count" with namespaces and filter
     operatorMetrics:
       # enabled configures whether operator metrics should be enabled or not

documentation/src/main/resources/pages/ditto/installation-operating.md

@@ -340,18 +340,18 @@ ditto {
   caching-signal-enrichment-facade-provider = org.eclipse.ditto.thingsearch.service.persistence.write.streaming.SearchIndexingSignalEnrichmentFacadeProvider
   //...
   search {
-    namespace-search-include-fields = [
+    namespace-indexed-fields = [
       {
         namespace-pattern = "org.eclipse.test"
-        search-include-fields = [
+        indexed-fields = [
            "attributes",
            "features/info/properties",
            "features/info/other"
         ]
       },
       {
         namespace-pattern = "org.eclipse*"
-        search-include-fields = [
+        indexed-fields = [
            "attributes",
            "features/info"
         ]
@@ -378,13 +378,13 @@ Important notes:
 Example for configuring the same configuration via system properties for the `things-search` service:
 
 ```shell
--Dditto.search.namespace-search-include-fields.0.namespace-pattern=org.eclipse.test
--Dditto.search.namespace-search-include-fields.0.search-include-fields.0=attributes
--Dditto.search.namespace-search-include-fields.0.search-include-fields.1=features/info/properties
--Dditto.search.namespace-search-include-fields.0.search-include-fields.2=features/info/other
--Dditto.search.namespace-search-include-fields.1.namespace-pattern=org.eclipse*
--Dditto.search.namespace-search-include-fields.1.search-include-fields.0=attributes
--Dditto.search.namespace-search-include-fields.1.search-include-fields.1=features/info
+-Dditto.search.namespace-indexed-fields.0.namespace-pattern=org.eclipse.test
+-Dditto.search.namespace-indexed-fields.0.indexed-fields.0=attributes
+-Dditto.search.namespace-indexed-fields.0.indexed-fields.1=features/info/properties
+-Dditto.search.namespace-indexed-fields.0.indexed-fields.2=features/info/other
+-Dditto.search.namespace-indexed-fields.1.namespace-pattern=org.eclipse*
+-Dditto.search.namespace-indexed-fields.1.indexed-fields.0=attributes
+-Dditto.search.namespace-indexed-fields.1.indexed-fields.1=features/info
 ```
 
 ## Logging

thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DefaultNamespaceSearchIndexConfig.java

@@ -12,9 +12,6 @@
  */
 package org.eclipse.ditto.thingsearch.service.common.config;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 
@@ -30,25 +27,22 @@ public final class DefaultNamespaceSearchIndexConfig implements NamespaceSearchI
 
     private final String namespacePattern;
 
-    private final List<String> searchIncludeFields;
+    private final List<String> includedFields;
 
     private DefaultNamespaceSearchIndexConfig(final ConfigWithFallback configWithFallback) {
 
-        this.namespacePattern = configWithFallback.getString(NamespaceSearchIndexConfigValue.NAMESPACE_PATTERN.getConfigPath());
+        this.namespacePattern =
+                configWithFallback.getString(NamespaceSearchIndexConfigValue.NAMESPACE_PATTERN.getConfigPath());
 
-        final List<String> fields = configWithFallback.getStringList(NamespaceSearchIndexConfigValue.SEARCH_INCLUDE_FIELDS.getConfigPath());
+        final List<String> fields =
+                configWithFallback.getStringList(NamespaceSearchIndexConfigValue.INDEXED_FIELDS.getConfigPath());
         if (!fields.isEmpty()) {
-            this.searchIncludeFields = Collections.unmodifiableList(new ArrayList<>(fields));
+            this.includedFields = List.copyOf(fields);
         } else {
-            this.searchIncludeFields = List.of();
+            this.includedFields = List.of();
         }
     }
 
-    private DefaultNamespaceSearchIndexConfig(final String namespacePattern, final Collection<String> fields) {
-        this.namespacePattern = namespacePattern;
-        this.searchIncludeFields = Collections.unmodifiableList(new ArrayList<>(fields));
-    }
-
     /**
      * Returns an instance of {@code DefaultNamespaceSearchIndexConfig} based on the settings of the specified Config.
      *
@@ -66,8 +60,8 @@ public String getNamespacePattern() {
     }
 
     @Override
-    public List<String> getSearchIncludeFields() {
-        return searchIncludeFields;
+    public List<String> getIndexedFields() {
+        return includedFields;
     }
 
     @Override
@@ -79,19 +73,20 @@ public boolean equals(final Object o) {
             return false;
         }
         final DefaultNamespaceSearchIndexConfig that = (DefaultNamespaceSearchIndexConfig) o;
-        return Objects.equals(namespacePattern, that.namespacePattern) && searchIncludeFields.equals(that.searchIncludeFields);
+        return Objects.equals(namespacePattern, that.namespacePattern) &&
+                includedFields.equals(that.includedFields);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(namespacePattern, searchIncludeFields);
+        return Objects.hash(namespacePattern, includedFields);
     }
 
     @Override
     public String toString() {
         return getClass().getSimpleName() + " [" +
-                "namespace=" + namespacePattern +
-                ", searchIncludeFields=" + searchIncludeFields +
+                "namespacePattern=" + namespacePattern +
+                ", searchIncludeFields=" + includedFields +
                 "]";
     }
 }

thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/DittoSearchConfig.java

@@ -12,8 +12,13 @@
  */
 package org.eclipse.ditto.thingsearch.service.common.config;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.BiConsumer;
 import java.util.function.BinaryOperator;
 import java.util.function.Function;
@@ -25,13 +30,11 @@
 import javax.annotation.Nullable;
 import javax.annotation.concurrent.Immutable;
 
-import com.typesafe.config.*;
 import org.eclipse.ditto.base.service.config.DittoServiceConfig;
 import org.eclipse.ditto.base.service.config.http.HttpConfig;
 import org.eclipse.ditto.base.service.config.limits.LimitsConfig;
 import org.eclipse.ditto.internal.utils.cluster.config.ClusterConfig;
 import org.eclipse.ditto.internal.utils.config.ConfigWithFallback;
-import org.eclipse.ditto.internal.utils.config.KnownConfigValue;
 import org.eclipse.ditto.internal.utils.config.ScopedConfig;
 import org.eclipse.ditto.internal.utils.config.WithConfigPath;
 import org.eclipse.ditto.internal.utils.health.config.DefaultHealthCheckConfig;
@@ -45,6 +48,11 @@
 import org.eclipse.ditto.internal.utils.persistence.operations.PersistenceOperationsConfig;
 import org.eclipse.ditto.internal.utils.tracing.config.TracingConfig;
 
+import com.typesafe.config.Config;
+import com.typesafe.config.ConfigFactory;
+import com.typesafe.config.ConfigList;
+import com.typesafe.config.ConfigValue;
+
 
 /**
  * This class is the default implementation of {@link SearchConfig}.
@@ -65,8 +73,8 @@ public final class DittoSearchConfig implements SearchConfig, WithConfigPath {
     private final MongoDbConfig mongoDbConfig;
     private final SearchPersistenceConfig queryPersistenceConfig;
     private final Map<String, String> simpleFieldMappings;
+    private final List<NamespaceSearchIndexConfig> namespaceIndexedFields;
     private final DefaultOperatorMetricsConfig operatorMetricsConfig;
-    private final List<NamespaceSearchIndexConfig> namespaceSearchIncludeFields;
 
     private DittoSearchConfig(final ScopedConfig dittoScopedConfig) {
         dittoServiceConfig = DittoServiceConfig.of(dittoScopedConfig, CONFIG_PATH);
@@ -86,8 +94,8 @@ private DittoSearchConfig(final ScopedConfig dittoScopedConfig) {
         queryPersistenceConfig = DefaultSearchPersistenceConfig.of(queryConfig);
         simpleFieldMappings =
                 convertToMap(configWithFallback.getConfig(SearchConfigValue.SIMPLE_FIELD_MAPPINGS.getConfigPath()));
+        namespaceIndexedFields = loadNamespaceSearchIndexList(configWithFallback);
         operatorMetricsConfig = DefaultOperatorMetricsConfig.of(configWithFallback);
-        namespaceSearchIncludeFields = loadNamespaceSearchIndexList(configWithFallback);
     }
 
     /**
@@ -108,8 +116,8 @@ public Optional<String> getMongoHintsByNamespace() {
     }
 
     @Override
-    public List<NamespaceSearchIndexConfig> getNamespaceSearchIncludeFields() {
-        return namespaceSearchIncludeFields;
+    public List<NamespaceSearchIndexConfig> getNamespaceIndexedFields() {
+        return namespaceIndexedFields;
     }
 
     @Override
@@ -196,14 +204,14 @@ public boolean equals(final Object o) {
                 Objects.equals(queryPersistenceConfig, that.queryPersistenceConfig) &&
                 Objects.equals(simpleFieldMappings, that.simpleFieldMappings) &&
                 Objects.equals(operatorMetricsConfig, that.operatorMetricsConfig) &&
-                Objects.equals(namespaceSearchIncludeFields, that.namespaceSearchIncludeFields);
+                Objects.equals(namespaceIndexedFields, that.namespaceIndexedFields);
     }
 
     @Override
     public int hashCode() {
         return Objects.hash(mongoHintsByNamespace, updaterConfig, dittoServiceConfig, healthCheckConfig,
                 indexInitializationConfig, persistenceOperationsConfig, mongoDbConfig, queryPersistenceConfig,
-                simpleFieldMappings, operatorMetricsConfig, namespaceSearchIncludeFields);
+                simpleFieldMappings, operatorMetricsConfig, namespaceIndexedFields);
     }
 
     @Override
@@ -218,8 +226,8 @@ public String toString() {
                 ", mongoDbConfig=" + mongoDbConfig +
                 ", queryPersistenceConfig=" + queryPersistenceConfig +
                 ", simpleFieldMappings=" + simpleFieldMappings +
+                ", namespaceIndexedFields=" + namespaceIndexedFields +
                 ", operatorMetricsConfig=" + operatorMetricsConfig +
-                ", namespaceSearchIncludeFields=" + namespaceSearchIncludeFields +
                 "]";
     }
 
@@ -239,10 +247,10 @@ private static Map<String, String> convertToMap(final Config config) {
 
     private static List<NamespaceSearchIndexConfig> loadNamespaceSearchIndexList(final ConfigWithFallback config) {
 
-        final ConfigList namespaceSearchIndexConfig = config.getList(
-                SearchConfigValue.NAMESPACE_SEARCH_INCLUDE_FIELDS.getConfigPath());
+        final ConfigList namespaceIndexedFieldsConfig = config.getList(
+                SearchConfigValue.NAMESPACE_INDEXED_FIELDS.getConfigPath());
 
-        return namespaceSearchIndexConfig.stream().collect(NamespaceSearchIndexCollector.toNamespaceSearchIndexList());
+        return namespaceIndexedFieldsConfig.stream().collect(NamespaceSearchIndexCollector.toNamespaceSearchIndexList());
     }
 
     private static class NamespaceSearchIndexCollector implements
@@ -266,12 +274,12 @@ public BiConsumer<List<NamespaceSearchIndexConfig>, ConfigValue> accumulator() {
         @Override
         public BinaryOperator<List<NamespaceSearchIndexConfig>> combiner() {
             return (left, right) -> Stream.concat(left.stream(), right.stream())
-                    .collect(Collectors.toList());
+                    .toList();
         }
 
         @Override
         public Function<List<NamespaceSearchIndexConfig>, List<NamespaceSearchIndexConfig>> finisher() {
-            return list -> Collections.unmodifiableList(new ArrayList<>(list));
+            return List::copyOf;
         }
 
         @Override

thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/NamespaceSearchIndexConfig.java

@@ -12,10 +12,11 @@
  */
 package org.eclipse.ditto.thingsearch.service.common.config;
 
-import org.eclipse.ditto.internal.utils.config.KnownConfigValue;
+import java.util.List;
 
 import javax.annotation.concurrent.Immutable;
-import java.util.List;
+
+import org.eclipse.ditto.internal.utils.config.KnownConfigValue;
 
 /**
  * Provides configuration settings of the namespace-scoped search indexes.
@@ -34,21 +35,21 @@ public interface NamespaceSearchIndexConfig {
     /**
      * Returns a list of fields that will be explicitly included in the search index.
      *
-     * @return the search projection fields.
+     * @return the indexed fields.
      */
-    List<String> getSearchIncludeFields();
+    List<String> getIndexedFields();
 
     enum NamespaceSearchIndexConfigValue implements KnownConfigValue {
 
         /**
-         * The namespace value to apply the search indexed fields.
+         * The namespace pattern to apply the search indexed fields.
          */
         NAMESPACE_PATTERN("namespace-pattern", ""),
 
         /**
          * The list of fields that will be included in the search DB.
          */
-        SEARCH_INCLUDE_FIELDS("search-include-fields", List.of());
+        INDEXED_FIELDS("indexed-fields", List.of());
 
         private final String configPath;
         private final Object defaultValue;

thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/common/config/SearchConfig.java

@@ -72,7 +72,7 @@ public interface SearchConfig extends ServiceSpecificConfig, WithHealthCheckConf
      * @return the search projection fields.
      * @since 3.5.0
      */
-    List<NamespaceSearchIndexConfig> getNamespaceSearchIncludeFields();
+    List<NamespaceSearchIndexConfig> getNamespaceIndexedFields();
 
     /**
      * An enumeration of the known config path expressions and their associated default values for SearchConfig.
@@ -104,8 +104,10 @@ enum SearchConfigValue implements KnownConfigValue {
 
         /**
          * Any fields to include in the search index, scoped by namespace.
+         *
+         * @since 3.5.0
          */
-        NAMESPACE_SEARCH_INCLUDE_FIELDS("namespace-search-include-fields", Collections.emptyList());
+        NAMESPACE_INDEXED_FIELDS("namespace-indexed-fields", Collections.emptyList());
 
         private final String path;
         private final Object defaultValue;

thingsearch/service/src/main/java/org/eclipse/ditto/thingsearch/service/persistence/write/streaming/SearchIndexingSignalEnrichmentFacadeProvider.java

@@ -45,7 +45,7 @@
  */
 public final class SearchIndexingSignalEnrichmentFacadeProvider implements CachingSignalEnrichmentFacadeProvider {
 
-    private static final Set<JsonFieldDefinition<?>> REQUIRED_INDEXED_FIELDS = Set.of(
+    private static final List<JsonFieldDefinition<?>> REQUIRED_INDEXED_FIELDS = List.of(
             Thing.JsonFields.ID,
             Thing.JsonFields.POLICY_ID,
             Thing.JsonFields.NAMESPACE,
@@ -76,17 +76,17 @@ public CachingSignalEnrichmentFacade getSignalEnrichmentFacade(
         // Build a map of field selectors for the enrichment facade to use to quickly look up by Thing namespace.
         final List<Pair<Pattern, JsonFieldSelector>> namespaceAndFieldSelector = new ArrayList<>();
 
-        for (final NamespaceSearchIndexConfig namespaceConfig : searchConfig.getNamespaceSearchIncludeFields()) {
+        for (final NamespaceSearchIndexConfig namespaceConfig : searchConfig.getNamespaceIndexedFields()) {
 
-            if (!namespaceConfig.getSearchIncludeFields().isEmpty()) {
+            if (!namespaceConfig.getIndexedFields().isEmpty()) {
 
                 // Ensure the constructed JsonFieldSelector has the required fields needed for the search to work.
                 final Set<String> set = new LinkedHashSet<>();
                 set.addAll(REQUIRED_INDEXED_FIELDS.stream()
                         .map(JsonFieldDefinition::getPointer)
                         .map(JsonPointer::toString)
                         .toList());
-                set.addAll(namespaceConfig.getSearchIncludeFields());
+                set.addAll(namespaceConfig.getIndexedFields());
 
                 final List<String> searchIncludeFields = new ArrayList<>(set);