Merge branch 'main' into data_stream_router_processor

docs/changelog/95187.yaml

@@ -0,0 +1,5 @@
+pr: 95187
+summary: Fix privileges check failures by adding `allow_restricted_indices` flag
+area: Transform
+type: bug
+issues: []

docs/changelog/95190.yaml

@@ -0,0 +1,5 @@
+pr: 95190
+summary: Add missing `text_similarity` config support to the Inference Processor
+area: Machine Learning
+type: bug
+issues: []

rest-api-spec/src/main/resources/rest-api-spec/api/behavioral_analytics.post_event.json → rest-api-spec/src/main/resources/rest-api-spec/api/search_application.post_behavioral_analytics_event.json

@@ -1,5 +1,5 @@
 {
-  "behavioral_analytics.post_event": {
+  "search_application.post_behavioral_analytics_event": {
     "documentation": {
       "url": "http://todo.com/tbd",
       "description": "Creates a behavioral analytics event for existing collection."

server/src/main/java/org/elasticsearch/bootstrap/Elasticsearch.java

@@ -446,8 +446,6 @@ private static void shutdown() {
         var es = INSTANCE;
         try {
             IOUtils.close(es.node, es.spawner);
-            LoggerContext context = (LoggerContext) LogManager.getContext(false);
-            Configurator.shutdown(context);
             if (es.node != null && es.node.awaitClose(10, TimeUnit.SECONDS) == false) {
                 throw new IllegalStateException(
                     "Node didn't stop within 10 seconds. " + "Any outstanding requests or tasks might get killed."
@@ -459,6 +457,9 @@ private static void shutdown() {
             LogManager.getLogger(Elasticsearch.class).warn("Thread got interrupted while waiting for the node to shutdown.");
             Thread.currentThread().interrupt();
         } finally {
+            LoggerContext context = (LoggerContext) LogManager.getContext(false);
+            Configurator.shutdown(context);
+
             es.keepAliveLatch.countDown();
         }
     }

x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/ml/action/InferModelActionResponseTests.java

@@ -29,6 +29,8 @@
 import org.elasticsearch.xpack.core.ml.inference.results.TextEmbeddingResultsTests;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResults;
 import org.elasticsearch.xpack.core.ml.inference.results.TextExpansionResultsTests;
+import org.elasticsearch.xpack.core.ml.inference.results.TextSimilarityInferenceResults;
+import org.elasticsearch.xpack.core.ml.inference.results.TextSimilarityInferenceResultsTests;
 import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResults;
 import org.elasticsearch.xpack.core.ml.inference.results.WarningInferenceResultsTests;
 
@@ -43,14 +45,15 @@ public class InferModelActionResponseTests extends AbstractWireSerializingTestCa
 
     private static List<String> INFERENCE_RESULT_TYPES = List.of(
         ClassificationInferenceResults.NAME,
-        RegressionInferenceResults.NAME,
         NerResults.NAME,
-        TextEmbeddingResults.NAME,
-        PyTorchPassThroughResults.NAME,
         FillMaskResults.NAME,
-        WarningInferenceResults.NAME,
+        PyTorchPassThroughResults.NAME,
         QuestionAnsweringInferenceResults.NAME,
-        TextExpansionResults.NAME
+        RegressionInferenceResults.NAME,
+        TextEmbeddingResults.NAME,
+        TextExpansionResults.NAME,
+        TextSimilarityInferenceResults.NAME,
+        WarningInferenceResults.NAME
     );
 
     @Override
@@ -79,14 +82,15 @@ protected Response mutateInstance(Response instance) {
     private static InferenceResults randomInferenceResult(String resultType) {
         return switch (resultType) {
             case ClassificationInferenceResults.NAME -> ClassificationInferenceResultsTests.createRandomResults();
-            case RegressionInferenceResults.NAME -> RegressionInferenceResultsTests.createRandomResults();
             case NerResults.NAME -> NerResultsTests.createRandomResults();
-            case TextEmbeddingResults.NAME -> TextEmbeddingResultsTests.createRandomResults();
-            case PyTorchPassThroughResults.NAME -> PyTorchPassThroughResultsTests.createRandomResults();
             case FillMaskResults.NAME -> FillMaskResultsTests.createRandomResults();
-            case WarningInferenceResults.NAME -> WarningInferenceResultsTests.createRandomResults();
+            case PyTorchPassThroughResults.NAME -> PyTorchPassThroughResultsTests.createRandomResults();
             case QuestionAnsweringInferenceResults.NAME -> QuestionAnsweringInferenceResultsTests.createRandomResults();
+            case RegressionInferenceResults.NAME -> RegressionInferenceResultsTests.createRandomResults();
+            case TextEmbeddingResults.NAME -> TextEmbeddingResultsTests.createRandomResults();
             case TextExpansionResults.NAME -> TextExpansionResultsTests.createRandomResults();
+            case TextSimilarityInferenceResults.NAME -> TextSimilarityInferenceResultsTests.createRandomResults();
+            case WarningInferenceResults.NAME -> WarningInferenceResultsTests.createRandomResults();
             default -> throw new AssertionError("unexpected result type [" + resultType + "]");
         };
     }

x-pack/plugin/ent-search/qa/rest/build.gradle

@@ -7,7 +7,7 @@ dependencies {
 
 restResources {
   restApi {
-    include '_common', 'cluster', 'nodes', 'indices', 'index', 'search_application', 'behavioral_analytics'
+    include '_common', 'cluster', 'nodes', 'indices', 'index', 'search_application'
   }
 }
 

x-pack/plugin/ent-search/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/entsearch/90_behavioral_analytics_event_post.yml

@@ -1,12 +1,12 @@
 setup:
   - do:
-      behavioral_analytics.put:
+      search_application.put_behavioral_analytics:
         name: my-test-analytics-collection
 
 ---
 teardown:
   - do:
-      behavioral_analytics.delete:
+      search_application.delete_behavioral_analytics:
         name: my-test-analytics-collection
         ignore: 404
 
@@ -15,7 +15,7 @@ teardown:
 ---
 "Post page_view analytics event":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "page_view"
         body:
@@ -30,7 +30,7 @@ teardown:
 "Post page_view analytics event - Missing page.url":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "page_view"
         body:
@@ -43,7 +43,7 @@ teardown:
 ---
 "Post page_view analytics event - With document":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "page_view"
         body:
@@ -60,7 +60,7 @@ teardown:
 ---
 "Post page_view analytics event - With page title":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "page_view"
         body:
@@ -75,7 +75,7 @@ teardown:
 ---
 "Post page_view analytics event - With referrer":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "page_view"
         body:
@@ -91,7 +91,7 @@ teardown:
 ---
 "Post search analytics event":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search"
         body:
@@ -106,7 +106,7 @@ teardown:
 "Post search analytics event – Missing search query":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search"
         body:
@@ -119,7 +119,7 @@ teardown:
 ---
 "Post search analytics event - With sort order":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search"
         body:
@@ -135,7 +135,7 @@ teardown:
 ---
 "Post search analytics event - With sort name and direction":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search"
         body:
@@ -152,7 +152,7 @@ teardown:
 ---
 "Post search analytics event - With pagination":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search"
         body:
@@ -169,7 +169,7 @@ teardown:
 ---
 "Post search analytics event - With search application":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search"
         body:
@@ -184,7 +184,7 @@ teardown:
 ---
 "Post search analytics event - With search results":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search"
         body:
@@ -214,7 +214,7 @@ teardown:
 ---
 "Post search_click analytics event":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search_click"
         body:
@@ -232,7 +232,7 @@ teardown:
 ---
 "Post search_click analytics event - Page Only":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search_click"
         body:
@@ -248,7 +248,7 @@ teardown:
 ---
 "Post search_click analytics event - Document Only":
   - do:
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search_click"
         body:
@@ -266,7 +266,7 @@ teardown:
 "Post search_click analytics event – Missing search query":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search_click"
         body:
@@ -282,7 +282,7 @@ teardown:
 "Post search_click analytics event – Missing page url and document":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "search_click"
         body:
@@ -299,7 +299,7 @@ teardown:
 "Post analytics event - Analytics collection does not exist":
   - do:
       catch: "missing"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: test-nonexistent-analytics-collection
         event_type: "page_view"
         body:
@@ -314,7 +314,7 @@ teardown:
 "Post analytics event - Event type does not exist":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "nonexistent-event-type"
         body:
@@ -331,7 +331,7 @@ teardown:
 "Post page_view analytics event - Missing session.id":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "page_view"
         body:
@@ -345,7 +345,7 @@ teardown:
 "Post page_view analytics event - Missing user.id":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "page_view"
         body:
@@ -359,7 +359,7 @@ teardown:
 "Post analytics event - Unknown event field":
   - do:
       catch: "bad_request"
-      behavioral_analytics.post_event:
+      search_application.post_behavioral_analytics_event:
         collection_name: my-test-analytics-collection
         event_type: "nonexistent-event-type"
         body:

x-pack/plugin/ml/src/main/java/org/elasticsearch/xpack/ml/inference/ingest/InferenceProcessor.java

@@ -47,6 +47,8 @@
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextEmbeddingConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextExpansionConfigUpdate;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfig;
+import org.elasticsearch.xpack.core.ml.inference.trainedmodel.TextSimilarityConfigUpdate;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfig;
 import org.elasticsearch.xpack.core.ml.inference.trainedmodel.ZeroShotClassificationConfigUpdate;
 import org.elasticsearch.xpack.core.ml.job.messages.Messages;
@@ -308,15 +310,18 @@ InferenceConfigUpdate inferenceConfigUpdateFromMap(Map<String, Object> configMap
             } else if (configMap.containsKey(RegressionConfig.NAME.getPreferredName())) {
                 checkSupportedVersion(RegressionConfig.EMPTY_PARAMS);
                 return RegressionConfigUpdate.fromMap(valueMap);
-            } else if (configMap.containsKey(TextExpansionConfig.NAME)) {
-                checkNlpSupported(TextExpansionConfig.NAME);
-                return TextExpansionConfigUpdate.fromMap(valueMap);
             } else if (configMap.containsKey(TextClassificationConfig.NAME)) {
                 checkNlpSupported(TextClassificationConfig.NAME);
                 return TextClassificationConfigUpdate.fromMap(valueMap);
             } else if (configMap.containsKey(TextEmbeddingConfig.NAME)) {
                 checkNlpSupported(TextEmbeddingConfig.NAME);
                 return TextEmbeddingConfigUpdate.fromMap(valueMap);
+            } else if (configMap.containsKey(TextExpansionConfig.NAME)) {
+                checkNlpSupported(TextExpansionConfig.NAME);
+                return TextExpansionConfigUpdate.fromMap(valueMap);
+            } else if (configMap.containsKey(TextSimilarityConfig.NAME)) {
+                checkNlpSupported(TextSimilarityConfig.NAME);
+                return TextSimilarityConfigUpdate.fromMap(valueMap);
             } else if (configMap.containsKey(ZeroShotClassificationConfig.NAME)) {
                 checkNlpSupported(ZeroShotClassificationConfig.NAME);
                 return ZeroShotClassificationConfigUpdate.fromMap(valueMap);
@@ -333,8 +338,11 @@ InferenceConfigUpdate inferenceConfigUpdateFromMap(Map<String, Object> configMap
                         FillMaskConfig.NAME,
                         NerConfig.NAME,
                         PassThroughConfig.NAME,
+                        QuestionAnsweringConfig.NAME,
                         TextClassificationConfig.NAME,
                         TextEmbeddingConfig.NAME,
+                        TextExpansionConfigUpdate.NAME,
+                        TextSimilarityConfig.NAME,
                         ZeroShotClassificationConfig.NAME
                     )
                 );