From 95855043a3598ca57fbc6dff022e5c6274c55511 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Tue, 21 May 2024 09:38:10 +0200 Subject: [PATCH] Disallow new rollup jobs in clusters with no rollup usage. (#108624) This change will add logic to the put rollup api that fails if no rollup job is active and no rollup index exists in the cluster. The logic first check whether there is an active rollup persistent task if there are no active rollup persistent tasks, then it checks whether any rollup index exists. The latter check is an expensive check, but assuming that it only runs as part of the put rollup job api and only when there are no rollup jobs, this should be ok. All tests that invoke the put rollup job api will need to be adjusted to create a dummy index that has rollup mapping metadata. Otherwise, tests can't create a rollup job. Closes #108381 --- docs/build.gradle | 46 ++++++ docs/changelog/108624.yaml | 12 ++ docs/reference/rollup/apis/put-job.asciidoc | 4 + .../rollup/rollup-getting-started.asciidoc | 4 + x-pack/plugin/build.gradle | 7 +- .../ml/integration/DatafeedJobsRestIT.java | 20 +++ .../action/RollupUsageTransportAction.java | 9 +- .../action/TransportPutRollupJobAction.java | 56 +++++++- .../TransportPutRollupJobActionTests.java | 135 ++++++++++++++++++ .../rest-api-spec/test/rollup/delete_job.yml | 12 ++ .../rest-api-spec/test/rollup/get_jobs.yml | 13 ++ .../test/rollup/get_rollup_caps.yml | 16 +++ .../test/rollup/get_rollup_index_caps.yml | 17 +++ .../rest-api-spec/test/rollup/put_job.yml | 46 ++++++ .../test/rollup/rollup_search.yml | 12 ++ .../test/rollup/security_tests.yml | 12 ++ .../rest-api-spec/test/rollup/start_job.yml | 12 ++ .../rest-api-spec/test/rollup/stop_job.yml | 12 ++ .../xpack/restart/FullClusterRestartIT.java | 17 +++ .../elasticsearch/multi_node/RollupIT.java | 18 ++- 20 files changed, 470 insertions(+), 10 deletions(-) create mode 100644 docs/changelog/108624.yaml create mode 100644 x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobActionTests.java diff --git a/docs/build.gradle b/docs/build.gradle index 7ca4820eea1af..7bffba0a1957b 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -843,6 +843,18 @@ buildRestTests.setups['library'] = ''' ''' buildRestTests.setups['sensor_rollup_job'] = ''' + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: {} + - do: indices.create: index: sensor-1 @@ -893,6 +905,18 @@ buildRestTests.setups['sensor_rollup_job'] = ''' } ''' buildRestTests.setups['sensor_started_rollup_job'] = ''' + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: {} + - do: indices.create: index: sensor-1 @@ -967,6 +991,28 @@ buildRestTests.setups['sensor_started_rollup_job'] = ''' ''' buildRestTests.setups['sensor_index'] = ''' + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: + id: my-id + index_pattern: "dummy-index-*" + rollup_index: "dummy-rollup-index" + cron: "*/30 * * * * ?" + page_size: 1000 + groups: + date_histogram: + field: timestamp + fixed_interval: 1h + delay: 7d + - do: indices.create: index: sensor-1 diff --git a/docs/changelog/108624.yaml b/docs/changelog/108624.yaml new file mode 100644 index 0000000000000..0da1fd2902c03 --- /dev/null +++ b/docs/changelog/108624.yaml @@ -0,0 +1,12 @@ +pr: 108624 +summary: Disallow new rollup jobs in clusters with no rollup usage +area: Rollup +type: breaking +issues: + - 108381 +breaking: + title: Disallow new rollup jobs in clusters with no rollup usage + area: Rollup + details: The put rollup API will fail with an error when a rollup job is created in a cluster with no rollup usage + impact: Clusters with no rollup usage (either no rollup job or index) can not create new rollup jobs + notable: true diff --git a/docs/reference/rollup/apis/put-job.asciidoc b/docs/reference/rollup/apis/put-job.asciidoc index 2392d7204df3b..c52e7a042e0ca 100644 --- a/docs/reference/rollup/apis/put-job.asciidoc +++ b/docs/reference/rollup/apis/put-job.asciidoc @@ -8,6 +8,10 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Use <> instead."] +WARNING: From 8.15.0 invoking this API in a cluster with no rollup usage will fail with a message about Rollup's +deprecation and planned removal. A cluster either needs to contain a rollup job or a rollup index in order for this API +to be allowed to execute. + Creates a {rollup-job}. [[rollup-put-job-api-request]] diff --git a/docs/reference/rollup/rollup-getting-started.asciidoc b/docs/reference/rollup/rollup-getting-started.asciidoc index 704e8e8fad0e9..a2b3956c47f79 100644 --- a/docs/reference/rollup/rollup-getting-started.asciidoc +++ b/docs/reference/rollup/rollup-getting-started.asciidoc @@ -7,6 +7,10 @@ deprecated::[8.11.0,"Rollups will be removed in a future version. Please <> to <> instead."] +WARNING: From 8.15.0 invoking the put job API in a cluster with no rollup usage will fail with a message about Rollup's +deprecation and planned removal. A cluster either needs to contain a rollup job or a rollup index in order for the +put job API to be allowed to execute. + To use the Rollup feature, you need to create one or more "Rollup Jobs". These jobs run continuously in the background and rollup the index or indices that you specify, placing the rolled documents in a secondary index (also of your choosing). diff --git a/x-pack/plugin/build.gradle b/x-pack/plugin/build.gradle index 72e63b3255999..ff45cc9430633 100644 --- a/x-pack/plugin/build.gradle +++ b/x-pack/plugin/build.gradle @@ -136,12 +136,6 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> "ml/categorization_agg/Test categorization aggregation with poor settings", "categorize_text was changed in 8.3, but experimental prior to the change" ) - task.skipTest("rollup/delete_job/Test basic delete_job", "rollup was an experimental feature, also see #41227") - task.skipTest("rollup/delete_job/Test delete job twice", "rollup was an experimental feature, also see #41227") - task.skipTest("rollup/delete_job/Test delete running job", "rollup was an experimental feature, also see #41227") - task.skipTest("rollup/get_jobs/Test basic get_jobs", "rollup was an experimental feature, also see #41227") - task.skipTest("rollup/put_job/Test basic put_job", "rollup was an experimental feature, also see #41227") - task.skipTest("rollup/start_job/Test start job twice", "rollup was an experimental feature, also see #41227") task.skipTest("indices.freeze/30_usage/Usage stats on frozen indices", "#70192 -- the freeze index API is removed from 8.0") task.skipTest("indices.freeze/20_stats/Translog stats on frozen indices", "#70192 -- the freeze index API is removed from 8.0") task.skipTest("indices.freeze/10_basic/Basic", "#70192 -- the freeze index API is removed from 8.0") @@ -152,6 +146,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure { task -> task.skipTest("spatial/70_script_doc_values/diagonal length", "precision changed in 8.4.0") task.skipTest("spatial/70_script_doc_values/geoshape value", "error message changed in 8.9.0") task.skipTest("security/authz/14_cat_indices/Test empty request while single authorized index", "not supported for compatibility") + task.skipTestsByFilePattern("**/rollup/**", "The rollup yaml tests in the 7.x branch don't know how to fake a cluster with rollup usage") task.replaceValueInMatch("_type", "_doc") task.addAllowedWarningRegex("\\[types removal\\].*") diff --git a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java index 1f76fe97144a8..2fc9a80ae3679 100644 --- a/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java +++ b/x-pack/plugin/ml/qa/native-multi-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/ml/integration/DatafeedJobsRestIT.java @@ -637,6 +637,7 @@ public void testInsufficientSearchPrivilegesOnPutWithJob() { } public void testCreationOnPutWithRollup() throws Exception { + createDummyRollupIndex(); setupDataAccessRole("airline-data-aggs-rollup"); String jobId = "privs-put-job-rollup"; String datafeedId = "datafeed-" + jobId; @@ -1248,6 +1249,7 @@ public void testLookbackWithPipelineBucketAgg() throws Exception { } public void testLookbackOnlyGivenAggregationsWithHistogramAndRollupIndex() throws Exception { + createDummyRollupIndex(); String jobId = "aggs-histogram-rollup-job"; Request createJobRequest = new Request("PUT", MachineLearning.BASE_PATH + "anomaly_detectors/" + jobId); createJobRequest.setJsonEntity(""" @@ -1351,6 +1353,7 @@ public void testLookbackOnlyGivenAggregationsWithHistogramAndRollupIndex() throw } public void testLookbackWithoutPermissionsAndRollup() throws Exception { + createDummyRollupIndex(); setupFullAccessRole("airline-data-aggs-rollup"); String jobId = "rollup-permission-test-network-job"; String datafeedId = "datafeed-" + jobId; @@ -1878,4 +1881,21 @@ private Response createJobAndDataFeed(String jobId, String datafeedId) throws IO .setAuthHeader(BASIC_AUTH_VALUE_ML_ADMIN_WITH_SOME_DATA_ACCESS) .build(); } + + private static void createDummyRollupIndex() throws IOException { + // create dummy rollup index to circumvent the check that prohibits rollup usage in empty clusters: + Request req = new Request("PUT", "dummy-rollup-index"); + req.setJsonEntity(""" + { + "mappings":{ + "_meta": { + "_rollup":{ + "my-id": {} + } + } + } + } + """); + client().performRequest(req); + } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java index c711553c99a17..4d3a9ef933255 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/RollupUsageTransportAction.java @@ -51,12 +51,17 @@ protected void masterOperation( ClusterState state, ActionListener listener ) { + int numberOfRollupJobs = findNumberOfRollupJobs(state); + RollupFeatureSetUsage usage = new RollupFeatureSetUsage(numberOfRollupJobs); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } + + static int findNumberOfRollupJobs(ClusterState state) { int numberOfRollupJobs = 0; PersistentTasksCustomMetadata persistentTasks = state.metadata().custom(PersistentTasksCustomMetadata.TYPE); if (persistentTasks != null) { numberOfRollupJobs = persistentTasks.findTasks(RollupJob.NAME, Predicates.always()).size(); } - RollupFeatureSetUsage usage = new RollupFeatureSetUsage(numberOfRollupJobs); - listener.onResponse(new XPackUsageFeatureResponse(usage)); + return numberOfRollupJobs; } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java index 9c3c34e2d63bd..b6f91c71445a4 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobAction.java @@ -30,12 +30,14 @@ import org.elasticsearch.cluster.block.ClusterBlockLevel; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; import org.elasticsearch.cluster.metadata.MappingMetadata; +import org.elasticsearch.cluster.metadata.Metadata; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.logging.DeprecationCategory; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.util.concurrent.EsExecutors; +import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.core.TimeValue; import org.elasticsearch.persistent.PersistentTasksCustomMetadata; @@ -45,6 +47,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.TransportService; import org.elasticsearch.xcontent.XContentBuilder; +import org.elasticsearch.xcontent.XContentParser; +import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; import org.elasticsearch.xpack.core.ClientHelper; import org.elasticsearch.xpack.core.XPackPlugin; @@ -56,12 +60,18 @@ import java.io.IOException; import java.util.Map; import java.util.Objects; +import java.util.Set; import static org.elasticsearch.xpack.core.ClientHelper.assertNoAuthorizationHeader; public class TransportPutRollupJobAction extends AcknowledgedTransportMasterNodeAction { private static final Logger LOGGER = LogManager.getLogger(TransportPutRollupJobAction.class); + private static final XContentParserConfiguration PARSER_CONFIGURATION = XContentParserConfiguration.EMPTY.withFiltering( + Set.of("_doc._meta._rollup"), + null, + false + ); private final PersistentTasksService persistentTasksService; private final Client client; @@ -102,6 +112,24 @@ protected void masterOperation( XPackPlugin.checkReadyForXPackCustomMetadata(clusterState); checkForDeprecatedTZ(request); + int numberOfCurrentRollupJobs = RollupUsageTransportAction.findNumberOfRollupJobs(clusterState); + if (numberOfCurrentRollupJobs == 0) { + try { + boolean hasRollupIndices = hasRollupIndices(clusterState.getMetadata()); + if (hasRollupIndices == false) { + listener.onFailure( + new IllegalArgumentException( + "new rollup jobs are not allowed in clusters that don't have any rollup usage, since rollup has been deprecated" + ) + ); + return; + } + } catch (IOException e) { + listener.onFailure(e); + return; + } + } + FieldCapabilitiesRequest fieldCapsRequest = new FieldCapabilitiesRequest().indices(request.indices()) .fields(request.getConfig().getAllFields().toArray(new String[0])); fieldCapsRequest.setParentTask(clusterService.localNode().getId(), task.getId()); @@ -180,7 +208,7 @@ static void createIndex( ); } - private static XContentBuilder createMappings(RollupJobConfig config) throws IOException { + static XContentBuilder createMappings(RollupJobConfig config) throws IOException { return XContentBuilder.builder(XContentType.JSON.xContent()) .startObject() .startObject("mappings") @@ -339,6 +367,32 @@ public void onTimeout(TimeValue timeout) { ); } + static boolean hasRollupIndices(Metadata metadata) throws IOException { + // Sniffing logic instead of invoking sourceAsMap(), which would materialize the entire mapping as map of maps. + for (var imd : metadata) { + if (imd.mapping() == null) { + continue; + } + + try (var parser = XContentHelper.createParser(PARSER_CONFIGURATION, imd.mapping().source().compressedReference())) { + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + if ("_doc".equals(parser.nextFieldName())) { + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + if ("_meta".equals(parser.nextFieldName())) { + if (parser.nextToken() == XContentParser.Token.START_OBJECT) { + if ("_rollup".equals(parser.nextFieldName())) { + return true; + } + } + } + } + } + } + } + } + return false; + } + @Override protected ClusterBlockException checkBlock(PutRollupJobAction.Request request, ClusterState state) { return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_WRITE); diff --git a/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobActionTests.java b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobActionTests.java new file mode 100644 index 0000000000000..017924e461e55 --- /dev/null +++ b/x-pack/plugin/rollup/src/test/java/org/elasticsearch/xpack/rollup/action/TransportPutRollupJobActionTests.java @@ -0,0 +1,135 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.rollup.action; + +import org.elasticsearch.cluster.metadata.IndexMetadata; +import org.elasticsearch.cluster.metadata.Metadata; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.IndexVersion; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class TransportPutRollupJobActionTests extends ESTestCase { + + public void testHasRollupIndices() throws IOException { + { + String mappings = """ + { + "_doc": { + "properties": { + "field1": { + "type": "long" + } + }, + "_meta": { + "_rollup": { + "job_id": {} + } + } + } + } + """; + var metadata = createMetadata(mappings); + assertTrue(TransportPutRollupJobAction.hasRollupIndices(metadata)); + } + { + String mappings = """ + { + "_doc": { + "properties": { + "field1": { + "type": "long" + } + }, + "_meta": { + "_rollup": { + "job_id": {} + } + } + } + } + """; + var metadata = createMetadata(mappings); + assertTrue(TransportPutRollupJobAction.hasRollupIndices(metadata)); + } + { + String mappings = """ + { + "_doc": { + "properties": { + "field1": { + "type": "long" + } + }, + "_meta": { + "other_metadata": {}, + "_rollup": { + "job_id": {} + } + } + } + } + """; + var metadata = createMetadata(mappings); + assertTrue(TransportPutRollupJobAction.hasRollupIndices(metadata)); + } + { + String mappings = """ + { + "_doc": { + "properties": { + "field1": { + "type": "long" + } + } + } + } + """; + var metadata = createMetadata(mappings); + assertFalse(TransportPutRollupJobAction.hasRollupIndices(metadata)); + } + { + String mappings = """ + { + "_doc": { + } + } + """; + var metadata = createMetadata(mappings); + assertFalse(TransportPutRollupJobAction.hasRollupIndices(metadata)); + } + { + String mappings = """ + { + "_doc": { + "properties": { + "field1": { + "type": "long" + } + }, + "_meta": { + "other_metadata": {} + } + } + } + """; + var metadata = createMetadata(mappings); + assertFalse(TransportPutRollupJobAction.hasRollupIndices(metadata)); + } + } + + private static Metadata createMetadata(String mappings) { + Settings.Builder b = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersion.current()); + var metadata = Metadata.builder() + .put(IndexMetadata.builder("my-rollup-index").settings(b).numberOfShards(1).numberOfReplicas(0).putMapping(mappings)) + .build(); + return metadata; + } + +} diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/delete_job.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/delete_job.yml index b95f518158ed6..bf1a91b5c81fa 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/delete_job.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/delete_job.yml @@ -12,6 +12,18 @@ setup: value_field: type: integer + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_jobs.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_jobs.yml index 671fb24715631..ff99c39ef9afc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_jobs.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_jobs.yml @@ -12,6 +12,19 @@ setup: value_field: type: integer + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + + --- "Test basic get_jobs": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_caps.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_caps.yml index 42acd41097bf2..834141343dcbc 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_caps.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_caps.yml @@ -33,6 +33,18 @@ setup: type: date value_field: type: integer + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser @@ -58,6 +70,10 @@ setup: ] } + - do: + indices.delete: + index: dummy-rollup-index + --- "Verify one job caps": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml index d1bbc5acf31bc..dca96eb325b87 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/get_rollup_index_caps.yml @@ -33,6 +33,19 @@ setup: type: date value_field: type: integer + + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser @@ -58,6 +71,10 @@ setup: ] } + - do: + indices.delete: + index: dummy-rollup-index + --- "Verify one job caps by rollup index": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/put_job.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/put_job.yml index dd301c0a29f4f..d45c13a2b8adb 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/put_job.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/put_job.yml @@ -26,6 +26,19 @@ setup: value_field: type: integer + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + + --- "Test basic put_job": - do: @@ -247,6 +260,39 @@ setup: ] } +--- +"Deprecation validation failure": + + - do: + indices.delete: + index: dummy-rollup-index + + - do: + catch: /new rollup jobs are not allowed in clusters that don't have any rollup usage, since rollup has been deprecated/ + headers: + Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser + rollup.put_job: + id: foo + body: > + { + "index_pattern": "foo", + "rollup_index": "foo_rollup", + "cron": "*/30 * * * * ?", + "page_size" :10, + "groups" : { + "date_histogram": { + "field": "the_field", + "calendar_interval": "1h" + } + }, + "metrics": [ + { + "field": "field_doesnt_exist", + "metrics": ["min", "max", "sum"] + } + ] + } + --- "Unknown Metric": diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/rollup_search.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/rollup_search.yml index c4aabc520ab5e..d3f21f16c3a30 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/rollup_search.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/rollup_search.yml @@ -14,6 +14,18 @@ setup: price: type: integer + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/security_tests.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/security_tests.yml index de961e3bcadcc..5bae9469a3c47 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/security_tests.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/security_tests.yml @@ -6,6 +6,18 @@ setup: cluster.health: wait_for_status: yellow + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + --- teardown: - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/start_job.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/start_job.yml index 3ff1c1bb6b4d7..50e6c46016348 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/start_job.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/start_job.yml @@ -12,6 +12,18 @@ setup: value_field: type: integer + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/stop_job.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/stop_job.yml index 93e07a1e07cc8..187c190a9efef 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/stop_job.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/rollup/stop_job.yml @@ -12,6 +12,18 @@ setup: value_field: type: integer + - do: + indices.create: + index: dummy-rollup-index + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + _meta: + _rollup: + my-id: { } + - do: headers: Authorization: "Basic eF9wYWNrX3Jlc3RfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" # run as x_pack_rest_user, i.e. the test setup superuser diff --git a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 9c9ebf24bda87..e8da9d1887799 100644 --- a/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/javaRestTest/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -433,6 +433,23 @@ public void testApiKeySuperuser() throws IOException { */ public void testRollupAfterRestart() throws Exception { if (isRunningAgainstOldCluster()) { + // create dummy rollup index to circumvent the check that prohibits rollup usage in empty clusters: + { + Request req = new Request("PUT", "dummy-rollup-index"); + req.setJsonEntity(""" + { + "mappings":{ + "_meta": { + "_rollup":{ + "my-id": {} + } + } + } + } + """); + client().performRequest(req); + } + final int numDocs = 59; final int year = randomIntBetween(1970, 2018); diff --git a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java index 91e273f7e34b4..e8fce8e513165 100644 --- a/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java +++ b/x-pack/qa/multi-node/src/javaRestTest/java/org/elasticsearch/multi_node/RollupIT.java @@ -69,9 +69,25 @@ private Settings getClientSettings(final String username, final String password) } public void testBigRollup() throws Exception { + // create dummy rollup index to circumvent the check that prohibits rollup usage in empty clusters: + { + Request req = new Request("PUT", "dummy-rollup-index"); + req.setJsonEntity(""" + { + "mappings":{ + "_meta": { + "_rollup":{ + "my-id": {} + } + } + } + } + """); + client().performRequest(req); + } + final int numDocs = 200; String dateFormat = "strict_date_optional_time"; - // create the test-index index try (XContentBuilder builder = jsonBuilder()) { builder.startObject();