Merge branch 'main' into certutil-csr-pass

modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/150_tsdb.yml

@@ -196,9 +196,10 @@ index without timestamp with pipeline:
 dynamic templates:
   - skip:
       version: " - 8.12.99"
-      features: "default_shards"
       reason: "Support for dynamic fields was added in 8.13"
   - do:
+      allowed_warnings:
+        - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
       indices.put_index_template:
         name: my-dynamic-template
         body:
@@ -326,9 +327,10 @@ dynamic templates:
 dynamic templates - conflicting aliases:
   - skip:
       version: " - 8.12.99"
-      features: "default_shards"
       reason: "Support for dynamic fields was added in 8.13"
   - do:
+      allowed_warnings:
+        - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
       indices.put_index_template:
         name: my-dynamic-template
         body:
@@ -422,9 +424,10 @@ dynamic templates - conflicting aliases:
 dynamic templates with nesting:
   - skip:
       version: " - 8.12.99"
-      features: "default_shards"
       reason: "Support for dynamic fields was added in 8.13"
   - do:
+      allowed_warnings:
+        - "index template [my-dynamic-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-dynamic-template] will take precedence during new index creation"
       indices.put_index_template:
         name: my-dynamic-template
         body:
@@ -561,6 +564,8 @@ subobject in passthrough object auto flatten:
       version: " - 8.12.99"
       reason: "Support for passthrough fields was added in 8.13"
   - do:
+      allowed_warnings:
+        - "index template [my-passthrough-template] has index patterns [k9s*] matching patterns from existing older templates [global] with patterns (global => [*]); this template [my-passthrough-template] will take precedence during new index creation"
       indices.put_index_template:
         name: my-passthrough-template
         body:

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/action/UpdateJobAction.java

@@ -31,7 +31,7 @@ private UpdateJobAction() {
     public static class Request extends AcknowledgedRequest<UpdateJobAction.Request> implements ToXContentObject {
 
         public static UpdateJobAction.Request parseRequest(String jobId, XContentParser parser) {
-            JobUpdate update = JobUpdate.EXTERNAL_PARSER.apply(parser, null).setJobId(jobId).build();
+            JobUpdate update = JobUpdate.PARSER.apply(parser, null).setJobId(jobId).build();
             return new UpdateJobAction.Request(jobId, update);
         }
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdate.java

@@ -34,57 +34,43 @@ public class JobUpdate implements Writeable, ToXContentObject {
     public static final ParseField DETECTORS = new ParseField("detectors");
     public static final ParseField CLEAR_JOB_FINISH_TIME = new ParseField("clear_job_finish_time");
 
-    // For internal updates
-    static final ConstructingObjectParser<Builder, Void> INTERNAL_PARSER = new ConstructingObjectParser<>(
-        "job_update",
-        args -> new Builder((String) args[0])
-    );
-
     // For parsing REST requests
-    public static final ConstructingObjectParser<Builder, Void> EXTERNAL_PARSER = new ConstructingObjectParser<>(
+    public static final ConstructingObjectParser<Builder, Void> PARSER = new ConstructingObjectParser<>(
         "job_update",
         args -> new Builder((String) args[0])
     );
 
     static {
-        for (ConstructingObjectParser<Builder, Void> parser : Arrays.asList(INTERNAL_PARSER, EXTERNAL_PARSER)) {
-            parser.declareString(ConstructingObjectParser.optionalConstructorArg(), Job.ID);
-            parser.declareStringArray(Builder::setGroups, Job.GROUPS);
-            parser.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION);
-            parser.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS);
-            parser.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.STRICT_PARSER, Job.MODEL_PLOT_CONFIG);
-            parser.declareObject(Builder::setAnalysisLimits, AnalysisLimits.STRICT_PARSER, Job.ANALYSIS_LIMITS);
-            parser.declareString(
-                (builder, val) -> builder.setBackgroundPersistInterval(
-                    TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())
-                ),
-                Job.BACKGROUND_PERSIST_INTERVAL
-            );
-            parser.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
-            parser.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS);
-            parser.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS);
-            parser.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
-            parser.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS);
-            parser.declareObject(
-                Builder::setPerPartitionCategorizationConfig,
-                PerPartitionCategorizationConfig.STRICT_PARSER,
-                AnalysisConfig.PER_PARTITION_CATEGORIZATION
-            );
-            parser.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT);
-            parser.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN);
-            parser.declareString(
-                (builder, val) -> builder.setModelPruneWindow(
-                    TimeValue.parseTimeValue(val, AnalysisConfig.MODEL_PRUNE_WINDOW.getPreferredName())
-                ),
-                AnalysisConfig.MODEL_PRUNE_WINDOW
-            );
-        }
-        // These fields should not be set by a REST request
-        INTERNAL_PARSER.declareString(Builder::setModelSnapshotId, Job.MODEL_SNAPSHOT_ID);
-        INTERNAL_PARSER.declareString(Builder::setModelSnapshotMinVersion, Job.MODEL_SNAPSHOT_MIN_VERSION);
-        INTERNAL_PARSER.declareString(Builder::setJobVersion, Job.JOB_VERSION);
-        INTERNAL_PARSER.declareBoolean(Builder::setClearFinishTime, CLEAR_JOB_FINISH_TIME);
-        INTERNAL_PARSER.declareObject(Builder::setBlocked, Blocked.STRICT_PARSER, Job.BLOCKED);
+        PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), Job.ID);
+        PARSER.declareStringArray(Builder::setGroups, Job.GROUPS);
+        PARSER.declareStringOrNull(Builder::setDescription, Job.DESCRIPTION);
+        PARSER.declareObjectArray(Builder::setDetectorUpdates, DetectorUpdate.PARSER, DETECTORS);
+        PARSER.declareObject(Builder::setModelPlotConfig, ModelPlotConfig.STRICT_PARSER, Job.MODEL_PLOT_CONFIG);
+        PARSER.declareObject(Builder::setAnalysisLimits, AnalysisLimits.STRICT_PARSER, Job.ANALYSIS_LIMITS);
+        PARSER.declareString(
+            (builder, val) -> builder.setBackgroundPersistInterval(
+                TimeValue.parseTimeValue(val, Job.BACKGROUND_PERSIST_INTERVAL.getPreferredName())
+            ),
+            Job.BACKGROUND_PERSIST_INTERVAL
+        );
+        PARSER.declareLong(Builder::setRenormalizationWindowDays, Job.RENORMALIZATION_WINDOW_DAYS);
+        PARSER.declareLong(Builder::setResultsRetentionDays, Job.RESULTS_RETENTION_DAYS);
+        PARSER.declareLong(Builder::setModelSnapshotRetentionDays, Job.MODEL_SNAPSHOT_RETENTION_DAYS);
+        PARSER.declareLong(Builder::setDailyModelSnapshotRetentionAfterDays, Job.DAILY_MODEL_SNAPSHOT_RETENTION_AFTER_DAYS);
+        PARSER.declareStringArray(Builder::setCategorizationFilters, AnalysisConfig.CATEGORIZATION_FILTERS);
+        PARSER.declareObject(
+            Builder::setPerPartitionCategorizationConfig,
+            PerPartitionCategorizationConfig.STRICT_PARSER,
+            AnalysisConfig.PER_PARTITION_CATEGORIZATION
+        );
+        PARSER.declareField(Builder::setCustomSettings, (p, c) -> p.map(), Job.CUSTOM_SETTINGS, ObjectParser.ValueType.OBJECT);
+        PARSER.declareBoolean(Builder::setAllowLazyOpen, Job.ALLOW_LAZY_OPEN);
+        PARSER.declareString(
+            (builder, val) -> builder.setModelPruneWindow(
+                TimeValue.parseTimeValue(val, AnalysisConfig.MODEL_PRUNE_WINDOW.getPreferredName())
+            ),
+            AnalysisConfig.MODEL_PRUNE_WINDOW
+        );
     }
 
     private final String jobId;

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/job/config/JobUpdateTests.java

@@ -14,7 +14,6 @@
 import org.elasticsearch.core.TimeValue;
 import org.elasticsearch.test.AbstractXContentSerializingTestCase;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ml.MlConfigVersion;
 import org.elasticsearch.xpack.core.ml.job.process.autodetect.state.ModelSnapshot;
 import org.elasticsearch.xpack.core.ml.utils.MlConfigVersionUtils;
 
@@ -35,8 +34,6 @@
 
 public class JobUpdateTests extends AbstractXContentSerializingTestCase<JobUpdate> {
 
-    private boolean useInternalParser = randomBoolean();
-
     @Override
     protected JobUpdate createTestInstance() {
         return createRandom(randomAlphaOfLength(4), null);
@@ -49,7 +46,7 @@ protected JobUpdate mutateInstance(JobUpdate instance) {
 
     /**
      * Creates a completely random update when the job is null
-     * or a random update that is is valid for the given job
+     * or a random update that is valid for the given job
      */
     public JobUpdate createRandom(String jobId, @Nullable Job job) {
         JobUpdate.Builder update = new JobUpdate.Builder(jobId);
@@ -126,24 +123,9 @@ public JobUpdate createRandom(String jobId, @Nullable Job job) {
         if (randomBoolean()) {
             update.setCustomSettings(Collections.singletonMap(randomAlphaOfLength(10), randomAlphaOfLength(10)));
         }
-        if (useInternalParser && randomBoolean()) {
-            update.setModelSnapshotId(randomAlphaOfLength(10));
-        }
-        if (useInternalParser && randomBoolean()) {
-            update.setModelSnapshotMinVersion(MlConfigVersion.CURRENT);
-        }
-        if (useInternalParser && randomBoolean()) {
-            update.setJobVersion(MlConfigVersionUtils.randomCompatibleVersion(random()));
-        }
-        if (useInternalParser) {
-            update.setClearFinishTime(randomBoolean());
-        }
         if (randomBoolean()) {
             update.setAllowLazyOpen(randomBoolean());
         }
-        if (useInternalParser && randomBoolean() && (job == null || job.isDeleting() == false)) {
-            update.setBlocked(BlockedTests.createRandom());
-        }
         if (randomBoolean() && job != null) {
             update.setModelPruneWindow(
                 TimeValue.timeValueSeconds(
@@ -251,11 +233,7 @@ protected Writeable.Reader<JobUpdate> instanceReader() {
 
     @Override
     protected JobUpdate doParseInstance(XContentParser parser) {
-        if (useInternalParser) {
-            return JobUpdate.INTERNAL_PARSER.apply(parser, null).build();
-        } else {
-            return JobUpdate.EXTERNAL_PARSER.apply(parser, null).build();
-        }
+        return JobUpdate.PARSER.apply(parser, null).build();
     }
 
     public void testMergeWithJob() {

x-pack/plugin/esql/qa/testFixtures/src/main/resources/spatial.csv-spec

@@ -115,6 +115,7 @@ stXFromAirportsSupportsNull#[skip:-8.13.99, reason:st_x and st_y added in 8.14]
 FROM airports
 | EVAL x = FLOOR(ABS(ST_X(city_location))/200), y = FLOOR(ABS(ST_Y(city_location))/100)
 | STATS c = count(*) BY x, y
+| SORT c DESC
 ;
 
 c:long  | x:double  | y:double

x-pack/plugin/ml/src/test/java/org/elasticsearch/xpack/ml/datafeed/DatafeedNodeSelectorTests.java

@@ -313,6 +313,7 @@ public void testShardNotAllActive() {
             .checkDatafeedTaskCanBeCreated();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106107")
     public void testIndexDoesntExist() {
         Job job = createScheduledJob("job_id").build(new Date());
         DatafeedConfig df = createDatafeed("datafeed_id", job.getId(), Collections.singletonList("not_foo"));
@@ -497,6 +498,7 @@ public void testSelectNode_jobTaskStale() {
             .checkDatafeedTaskCanBeCreated();
     }
 
+    @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/106108")
     public void testSelectNode_GivenJobOpeningAndIndexDoesNotExist() {
         // Here we test that when there are 2 problems, the most critical gets reported first.
         // In this case job is Opening (non-critical) and the index does not exist (critical)

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/profiling/10_basic.yml

@@ -166,6 +166,9 @@ teardown:
 
 ---
 "Test flamegraph from profiling-events":
+  - skip:
+      reason: "https://github.com/elastic/elasticsearch/issues/106103"
+      version: "all"
   - do:
       profiling.flamegraph:
         body: >
@@ -192,6 +195,9 @@ teardown:
 
 ---
 "Test flamegraph from test-events":
+  - skip:
+      reason: "https://github.com/elastic/elasticsearch/issues/106103"
+      version: "all"
   - do:
       profiling.flamegraph:
         body: >