Do not copy index.default_pipeline and index.final_pipeline (elastic#…

…96494) (elastic#96500)

We need to skip copying the two settings, index.default_pipeline and index.final_pipeline,
from the source index to the target index in order to avoid pipeline processing to be applied 
to the target (downsampled) index. Some pipeline scripts might fail due to the differences in
mappings and settings between the downsampling source and target index.

docs/changelog/96494.yaml

@@ -0,0 +1,6 @@
+pr: 96494
+summary: Do not copy `index.default_pipeline` and `index.final_pipeline`
+area: Rollup
+type: bug
+issues:
+ - 96478

x-pack/plugin/rollup/qa/rest/build.gradle

@@ -17,7 +17,7 @@ dependencies {
 
 restResources {
   restApi {
-    include '_common', 'bulk', 'cluster', 'indices', 'search'
+    include '_common', 'bulk', 'cluster', 'indices', 'search', 'ingest.put_pipeline', 'ingest.delete_pipeline'
   }
 }
 

x-pack/plugin/rollup/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/rollup/60_settings.yml

@@ -0,0 +1,100 @@
+---
+"Downsample index with pipeline":
+  - skip:
+      version: " - 8.4.99"
+      reason: "rollup renamed to downsample in 8.5.0"
+
+  - do:
+      ingest.put_pipeline:
+        id: "pipeline"
+        body: >
+          {
+            "processors": [
+             {
+               "set" : {
+                 "field" : "field",
+                 "value": 42
+               }
+              }
+            ]
+          }
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+            index:
+              default_pipeline: "pipeline"
+              final_pipeline: "pipeline"
+              mode: time_series
+              routing_path: [metricset, k8s.pod.uid]
+              time_series:
+                start_time: 2021-04-28T00:00:00Z
+                end_time: 2021-04-29T00:00:00Z
+          mappings:
+            properties:
+              "@timestamp":
+                type: date
+              metricset:
+                type: keyword
+                time_series_dimension: true
+              value:
+                type: double
+                time_series_metric: gauge
+  - do:
+      bulk:
+        refresh: true
+        index: test
+        body:
+          - '{"index": {}}'
+          - '{"@timestamp": "2021-04-28T18:50:04.467Z", "metricset": "pod", "value": 10}'
+
+  - do:
+      search:
+        index: test
+        body:
+          query:
+            match_all: {}
+
+  - match: { hits.total.value: 1 }
+  - match: { hits.hits.0._source.field: 42 }
+
+  - do:
+      indices.put_settings:
+        index: test
+        body:
+          index.blocks.write: true
+
+  - do:
+      indices.downsample:
+        index: test
+        target_index: test-rollup
+        body: >
+          {
+            "fixed_interval": "1h"
+          }
+
+  - do:
+      indices.get_settings:
+        index: test
+
+  - match: { test.settings.index.default_pipeline: "pipeline" }
+  - match: { test.settings.index.final_pipeline: "pipeline" }
+
+  - do:
+      indices.get_settings:
+        index: test-rollup
+
+  - match: { test-rollup.settings.index.default_pipeline: null }
+  - match: { test-rollup.settings.index.final_pipeline: null }
+
+
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "pipeline"
+        ignore: 404

x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/downsample/TransportDownsampleAction.java

@@ -76,6 +76,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import static org.elasticsearch.index.mapper.TimeSeriesParams.TIME_SERIES_METRIC_PARAM;
 
@@ -97,6 +98,12 @@ public class TransportDownsampleAction extends AcknowledgedTransportMasterNodeAc
     private final IndexScopedSettings indexScopedSettings;
     private final ThreadContext threadContext;
 
+    private static final Set<String> FORBIDDEN_SETTINGS = Set.of(
+        IndexSettings.DEFAULT_PIPELINE.getKey(),
+        IndexSettings.FINAL_PIPELINE.getKey(),
+        IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.getKey()
+    );
+
     /**
      * This is the cluster state task executor for cluster state update actions.
      */
@@ -584,12 +591,15 @@ private IndexMetadata.Builder copyIndexMetadata(IndexMetadata sourceIndexMetadat
                 // the same rules with resize apply
                 continue;
             }
+            // Do not copy index settings which are valid for the source index but not for the target index
+            if (FORBIDDEN_SETTINGS.contains(key)) {
+                continue;
+            }
             // Do not override settings that have already been set in the rollup index.
-            // Also, we don't want to copy the `index.block.write` setting that we know
-            // it is set in the source index settings.
-            if (IndexMetadata.SETTING_BLOCKS_WRITE.equals(key) || targetSettings.keys().contains(key)) {
+            if (targetSettings.keys().contains(key)) {
                 continue;
             }
+
             targetSettings.copy(key, sourceIndexMetadata.getSettings());
         }