From 7d2c5acabf0257504d3a3cb2076ba92821da6220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:06:24 +0100 Subject: [PATCH 1/6] Mute failing Test flamegraph tests (#106106) --- .../resources/rest-api-spec/test/profiling/10_basic.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml index 367655ba89388..0948eb662f07f 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml @@ -166,6 +166,8 @@ teardown: --- "Test flamegraph from profiling-events": + - skip: + reason: "https://github.com/elastic/elasticsearch/issues/106103" - do: profiling.flamegraph: body: > @@ -192,6 +194,8 @@ teardown: --- "Test flamegraph from test-events": + - skip: + reason: "https://github.com/elastic/elasticsearch/issues/106103" - do: profiling.flamegraph: body: > From 179739effdad01e0f9e35b09799cec07603e05bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:10:28 +0100 Subject: [PATCH 2/6] Mute testIndexDoesntExist and testSelectNode_GivenJobOpeningAndIndexDoesNotExist (#106109) Mute: https://github.com/elastic/elasticsearch/issues/106107 and https://github.com/elastic/elasticsearch/issues/106108 --- .../xpack/ml/datafeed/DatafeedNodeSelectorTests.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java index 5ddba7519eef2..517c851d43804 100644 --- a/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java +++ b/x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java @@ -313,6 +313,7 @@ public void testShardNotAllActive() { .checkDatafeedTaskCanBeCreated(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106107") public void testIndexDoesntExist() { Job job = createScheduledJob("job_id").build(new Date()); DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("not_foo")); @@ -497,6 +498,7 @@ public void testSelectNode_jobTaskStale() { .checkDatafeedTaskCanBeCreated(); } + @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106108") public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() { // Here we test that when there are 2 problems, the most critical gets reported first. // In this case job is Opening (non-critical) and the index does not exist (critical) From 1530eb1bf782e56b1d1e9ddf30b2cf3098ae5d0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Fred=C3=A9n?= <109296772+jfreden@users.noreply.github.com> Date: Fri, 8 Mar 2024 10:41:55 +0100 Subject: [PATCH 3/6] Add version to skip block (#106111) Missed the version in the skip block --- .../resources/rest-api-spec/test/profiling/10_basic.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml index 0948eb662f07f..1e0c260a70e4f 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml @@ -168,6 +168,7 @@ teardown: "Test flamegraph from profiling-events": - skip: reason: "https://github.com/elastic/elasticsearch/issues/106103" + version: "all" - do: profiling.flamegraph: body: > @@ -196,6 +197,7 @@ teardown: "Test flamegraph from test-events": - skip: reason: "https://github.com/elastic/elasticsearch/issues/106103" + version: "all" - do: profiling.flamegraph: body: > From d502ade41a329d9017a865cd6433f107cfbdf8f7 Mon Sep 17 00:00:00 2001 From: Kostas Krikellas <131142368+kkrik-es@users.noreply.github.com> Date: Fri, 8 Mar 2024 12:24:11 +0200 Subject: [PATCH 4/6] [TEST] add allowed warnings in tsdb yaml (#106006) Fixes #105944 --- .../rest-api-spec/test/data_stream/150_tsdb.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml index 278c14c09a31a..20eb33ecefdee 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml @@ -196,9 +196,10 @@ index without timestamp with pipeline: dynamic templates: - skip: version: " - 8.12.99" - features: "default_shards" reason: "Support for dynamic fields was added in 8.13" - do: + allowed_warnings: + - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" indices.put_index_template: name: my-dynamic-template body: @@ -326,9 +327,10 @@ dynamic templates: dynamic templates - conflicting aliases: - skip: version: " - 8.12.99" - features: "default_shards" reason: "Support for dynamic fields was added in 8.13" - do: + allowed_warnings: + - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" indices.put_index_template: name: my-dynamic-template body: @@ -422,9 +424,10 @@ dynamic templates - conflicting aliases: dynamic templates with nesting: - skip: version: " - 8.12.99" - features: "default_shards" reason: "Support for dynamic fields was added in 8.13" - do: + allowed_warnings: + - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation" indices.put_index_template: name: my-dynamic-template body: @@ -561,6 +564,8 @@ subobject in passthrough object auto flatten: version: " - 8.12.99" reason: "Support for passthrough fields was added in 8.13" - do: + allowed_warnings: + - "index template [my-passthrough-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-passthrough-template] will take precedence during new index creation" indices.put_index_template: name: my-passthrough-template body: From 946bfb725d7a468e2d1943f5fd5e7500e852043a Mon Sep 17 00:00:00 2001 From: Jan Kuipers <148754765+jan-elastic@users.noreply.github.com> Date: Fri, 8 Mar 2024 12:18:50 +0100 Subject: [PATCH 5/6] Remove ununsed JobUpdate.INTERNAL_PARSER (#106112) * Remove ununsed JobUpdate.INTERNAL_PARSER * Fix compile errors --- .../xpack/core/ml/action/UpdateJobAction.java | 2 +- .../xpack/core/ml/job/config/JobUpdate.java | 76 ++++++++----------- .../core/ml/job/config/JobUpdateTests.java | 26 +------ 3 files changed, 34 insertions(+), 70 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java index 4e80fcab05e2f..15cd272d12b8b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java @@ -31,7 +31,7 @@ private UpdateJobAction() { public static class Request extends AcknowledgedRequest implements ToXContentObject { public static UpdateJobAction.Request parseRequest(String jobId, XContentParser parser) { - JobUpdate update = JobUpdate.EXTERNAL_PARSER.apply(parser, null).setJobId(jobId).build(); + JobUpdate update = JobUpdate.PARSER.apply(parser, null).setJobId(jobId).build(); return new UpdateJobAction.Request(jobId, update); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java index 3ba40c70d0701..4b11314b8bb43 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java @@ -34,57 +34,43 @@ public class JobUpdate implements Writeable, ToXContentObject { public static final ParseField DETECTORS = new ParseField("detectors"); public static final ParseField CLEAR_JOB_FINISH_TIME = new ParseField("clear_job_finish_time"); - // For internal updates - static final ConstructingObjectParser INTERNAL_PARSER = new ConstructingObjectParser<>( - "job_update", - args -> new Builder((String) args[0]) - ); - // For parsing REST requests - public static final ConstructingObjectParser EXTERNAL_PARSER = new ConstructingObjectParser<>( + public static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "job_update", args -> new Builder((String) args[0]) ); static { - for (ConstructingObjectParser parser : Arrays.asList(INTERNAL_PARSER, EXTERNAL_PARSER)) { - parser.declareString(ConstructingObjectParser.optionalConstructorArg(), Job.ID); - parser.declareStringArray(Builder::setGroups, Job.GROUPS); - parser.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION); - parser.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS); - parser.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.STRICT_PARSER, Job.MODEL_PLOT_CONFIG); - parser.declareObject(Builder::setAnalysisLimits, AnalysisLimits.STRICT_PARSER, Job.ANALYSIS_LIMITS); - parser.declareString( - (builder, val) -> builder.setBackgroundPersistInterval( - TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName()) - ), - Job.BACKGROUND_PERSIST_INTERVAL - ); - parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); - parser.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); - parser.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); - parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); - parser.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); - parser.declareObject( - Builder::setPerPartitionCategorizationConfig, - PerPartitionCategorizationConfig.STRICT_PARSER, - AnalysisConfig.PER_PARTITION_CATEGORIZATION - ); - parser.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT); - parser.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN); - parser.declareString( - (builder, val) -> builder.setModelPruneWindow( - TimeValue.parseTimeValue(val, AnalysisConfig.MODEL_PRUNE_WINDOW.getPreferredName()) - ), - AnalysisConfig.MODEL_PRUNE_WINDOW - ); - } - // These fields should not be set by a REST request - INTERNAL_PARSER.declareString(Builder::setModelSnapshotId, Job.MODEL_SNAPSHOT_ID); - INTERNAL_PARSER.declareString(Builder::setModelSnapshotMinVersion, Job.MODEL_SNAPSHOT_MIN_VERSION); - INTERNAL_PARSER.declareString(Builder::setJobVersion, Job.JOB_VERSION); - INTERNAL_PARSER.declareBoolean(Builder::setClearFinishTime, CLEAR_JOB_FINISH_TIME); - INTERNAL_PARSER.declareObject(Builder::setBlocked, Blocked.STRICT_PARSER, Job.BLOCKED); + PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), Job.ID); + PARSER.declareStringArray(Builder::setGroups, Job.GROUPS); + PARSER.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION); + PARSER.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS); + PARSER.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.STRICT_PARSER, Job.MODEL_PLOT_CONFIG); + PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.STRICT_PARSER, Job.ANALYSIS_LIMITS); + PARSER.declareString( + (builder, val) -> builder.setBackgroundPersistInterval( + TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName()) + ), + Job.BACKGROUND_PERSIST_INTERVAL + ); + PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS); + PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS); + PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS); + PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS); + PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS); + PARSER.declareObject( + Builder::setPerPartitionCategorizationConfig, + PerPartitionCategorizationConfig.STRICT_PARSER, + AnalysisConfig.PER_PARTITION_CATEGORIZATION + ); + PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT); + PARSER.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN); + PARSER.declareString( + (builder, val) -> builder.setModelPruneWindow( + TimeValue.parseTimeValue(val, AnalysisConfig.MODEL_PRUNE_WINDOW.getPreferredName()) + ), + AnalysisConfig.MODEL_PRUNE_WINDOW + ); } private final String jobId; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java index 09ff29f768dce..24a3a097e9e2d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.core.TimeValue; import org.elasticsearch.test.AbstractXContentSerializingTestCase; import org.elasticsearch.xcontent.XContentParser; -import org.elasticsearch.xpack.core.ml.MlConfigVersion; import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot; import org.elasticsearch.xpack.core.ml.utils.MlConfigVersionUtils; @@ -35,8 +34,6 @@ public class JobUpdateTests extends AbstractXContentSerializingTestCase { - private boolean useInternalParser = randomBoolean(); - @Override protected JobUpdate createTestInstance() { return createRandom(randomAlphaOfLength(4), null); @@ -49,7 +46,7 @@ protected JobUpdate mutateInstance(JobUpdate instance) { /** * Creates a completely random update when the job is null - * or a random update that is is valid for the given job + * or a random update that is valid for the given job */ public JobUpdate createRandom(String jobId, @Nullable Job job) { JobUpdate.Builder update = new JobUpdate.Builder(jobId); @@ -126,24 +123,9 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) { if (randomBoolean()) { update.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10))); } - if (useInternalParser && randomBoolean()) { - update.setModelSnapshotId(randomAlphaOfLength(10)); - } - if (useInternalParser && randomBoolean()) { - update.setModelSnapshotMinVersion(MlConfigVersion.CURRENT); - } - if (useInternalParser && randomBoolean()) { - update.setJobVersion(MlConfigVersionUtils.randomCompatibleVersion(random())); - } - if (useInternalParser) { - update.setClearFinishTime(randomBoolean()); - } if (randomBoolean()) { update.setAllowLazyOpen(randomBoolean()); } - if (useInternalParser && randomBoolean() && (job == null || job.isDeleting() == false)) { - update.setBlocked(BlockedTests.createRandom()); - } if (randomBoolean() && job != null) { update.setModelPruneWindow( TimeValue.timeValueSeconds( @@ -251,11 +233,7 @@ protected Writeable.Reader instanceReader() { @Override protected JobUpdate doParseInstance(XContentParser parser) { - if (useInternalParser) { - return JobUpdate.INTERNAL_PARSER.apply(parser, null).build(); - } else { - return JobUpdate.EXTERNAL_PARSER.apply(parser, null).build(); - } + return JobUpdate.PARSER.apply(parser, null).build(); } public void testMergeWithJob() { From 8e6a2268cb0dca4f69656175de5db8f7d3fb1c83 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 8 Mar 2024 12:27:03 +0100 Subject: [PATCH 6/6] Very rarely (once so far in CI) the STATS order is reversed (#106110) --- .../esql/qa/testFixtures/src/main/resources/spatial.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec index 1eb4d82b5fcc2..5c789cee0492f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec @@ -115,6 +115,7 @@ stXFromAirportsSupportsNull#[skip:-8.13.99, reason:st_x and st_y added in 8.14] FROM airports | EVAL x = FLOOR(ABS(ST_X(city_location))/200), y = FLOOR(ABS(ST_Y(city_location))/100) | STATS c = count(*) BY x, y +| SORT c DESC ; c:long | x:double | y:double