[ML] Semantic search endpoint (#90450)

Adds a {index}_semantic_search endpoint which first converts the query text into a dense vector
using a NLP text embedding model then performs a knn search against an index containing 
dense vectors created with the same embedding model.

docs/reference/search/semantic-search.asciidoc

@@ -0,0 +1,133 @@
+[[knn-search-api]]
+=== Semantic search API
+++++
+<titleabbrev>Semantic search</titleabbrev>
+++++
+
+experimental::[]
+Semantic search uses a text embedding NLP model to generate a dense vector from the input query string.
+The resulting dense vector is then used in a <Knn Search> against an index containing dense vectors
+created with the same text embedding model. The search results are semantically similar as learned
+by the model.
+
+////
+[source,console]
+----
+PUT my-index
+{
+  "mappings": {
+    "properties": {
+      "text_embedding": {
+        "type": "dense_vector",
+        "dims": 512,
+        "index": true,
+        "similarity": "cosine"
+      }
+    }
+  }
+}
+----
+////
+
+[source,console]
+----
+GET my-index/_semantic_search
+{
+  "query_string": "A picture of a snow capped mountain",
+  "model_id": "my-text-embedding-model",
+  "knn": {
+    "field": "text_embedding",
+    "k": 10,
+    "num_candidates": 100
+  }
+}
+----
+// TEST[skip:TBD]
+
+
+[[semantic-search-api-request]]
+==== {api-request-title}
+
+`GET <target>/_semantic_search`
+
+`POST <target>/_semantic_search`
+
+[[semantic-search-api-prereqs]]
+==== {api-prereq-title}
+
+* If the {es} {security-features} are enabled, you must have the `read`
+<<privileges-list-indices,index privilege>> for the target data stream, index,
+or alias.
+
+[[Semantic-search-api-desc]]
+==== {api-description-title}
+
+The Semantic search API uses a text embedding model to create a dense vector
+representation of the query string.
+
+
+[[Semantic-search-api-path-params]]
+==== {api-path-parms-title}
+
+`<target>`::
+(Optional, string) Comma-separated list of data streams, indices, and aliases
+to search. Supports wildcards (`*`). To search all data streams and indices,
+use `*` or `_all`.
+
+[role="child_attributes"]
+[[Semantic-search-api-query-params]]
+==== {api-query-parms-title}
+
+include::{es-repo-dir}/rest-api/common-parms.asciidoc[tag=routing]
+
+[role="child_attributes"]
+[[semantic-search-api-request-body]]
+==== {api-request-body-title}
+
+`model_id`::
+(Required, string)
+include::{es-repo-dir}/ml/ml-shared.asciidoc[tag=model-id]
+
+`query_string`::
+(Required, string) The input text to embed
+
+`knn`::
+(Required, object) Defines the kNN query to run.
++
+.Properties of `knn` object
+[%collapsible%open]
+====
+`field`::
+(Required, string) The name of the vector field to search against. Must be a
+<<index-vectors-knn-search, `dense_vector` field with indexing enabled>>.
+
+`k`::
+(Required, integer) Number of nearest neighbors to return as top hits. This
+value must be less than `num_candidates`.
+
+`num_candidates`::
+(Required, integer) The number of nearest neighbor candidates to consider per
+shard. Cannot exceed 10,000. {es} collects `num_candidates` results from each
+shard, then merges them to find the top `k` results. Increasing
+`num_candidates` tends to improve the accuracy of the final `k` results.
+====
+
+`filter`::
+(Optional, <<query-dsl,Query DSL object>>) Query to filter the documents that
+can match. The kNN search will return the top `k` documents that also match
+this filter. The value can be a single query or a list of queries. If `filter`
+is not provided, all documents are allowed to match.
+
+
+
+include::{es-repo-dir}/search/search.asciidoc[tag=docvalue-fields-def]
+include::{es-repo-dir}/search/search.asciidoc[tag=fields-param-def]
+include::{es-repo-dir}/search/search.asciidoc[tag=source-filtering-def]
+include::{es-repo-dir}/search/search.asciidoc[tag=stored-fields-def]
+
+[role="child_attributes"]
+[[semantic-search-api-response-body]]
+==== {api-response-body-title}
+
+A sementic search response has the same structure as a kNN search response.
+

rest-api-spec/src/main/resources/rest-api-spec/api/semantic_search.json

@@ -0,0 +1,40 @@
+{
+  "semantic_search":{
+    "documentation":{
+      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/semantic-search.html",
+      "description":"Semantic search API using dense vector similarity"
+    },
+    "stability":"experimental",
+    "visibility":"public",
+    "headers":{
+      "accept": [ "application/json"],
+      "content_type": ["application/json"]
+    },
+    "url":{
+      "paths":[
+        {
+          "path":"/{index}/_semantic_search",
+          "methods":[
+            "GET",
+            "POST"
+          ],
+          "parts":{
+            "index":{
+              "type":"list",
+              "description":"A comma-separated list of index names to search; use `_all` to perform the operation on all indices"
+            }
+          }
+        }
+      ]
+    },
+    "params": {
+      "routing":{
+        "type":"list",
+        "description":"A comma-separated list of specific routing values"
+      }
+    },
+    "body":{
+      "description":"The search definition"
+    }
+  }
+}

server/src/main/java/org/elasticsearch/search/vectors/KnnSearchBuilder.java

@@ -35,12 +35,12 @@
  */
 public class KnnSearchBuilder implements Writeable, ToXContentFragment, Rewriteable<KnnSearchBuilder> {
     private static final int NUM_CANDS_LIMIT = 10000;
-    static final ParseField FIELD_FIELD = new ParseField("field");
-    static final ParseField K_FIELD = new ParseField("k");
-    static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates");
-    static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector");
-    static final ParseField FILTER_FIELD = new ParseField("filter");
-    static final ParseField BOOST_FIELD = AbstractQueryBuilder.BOOST_FIELD;
+    public static final ParseField FIELD_FIELD = new ParseField("field");
+    public static final ParseField K_FIELD = new ParseField("k");
+    public static final ParseField NUM_CANDS_FIELD = new ParseField("num_candidates");
+    public static final ParseField QUERY_VECTOR_FIELD = new ParseField("query_vector");
+    public static final ParseField FILTER_FIELD = new ParseField("filter");
+    public static final ParseField BOOST_FIELD = AbstractQueryBuilder.BOOST_FIELD;
 
     private static final ConstructingObjectParser<KnnSearchBuilder, Void> PARSER = new ConstructingObjectParser<>("knn", args -> {
         @SuppressWarnings("unchecked")

server/src/test/java/org/elasticsearch/search/vectors/KnnSearchBuilderTests.java

@@ -32,6 +32,26 @@ public class KnnSearchBuilderTests extends AbstractXContentSerializingTestCase<K
     private NamedWriteableRegistry namedWriteableRegistry;
     private NamedXContentRegistry namedXContentRegistry;
 
+    public static KnnSearchBuilder randomTestInstance() {
+        String field = randomAlphaOfLength(6);
+        int dim = randomIntBetween(2, 30);
+        float[] vector = randomVector(dim);
+        int k = randomIntBetween(1, 100);
+        int numCands = randomIntBetween(k + 20, 1000);
+
+        KnnSearchBuilder builder = new KnnSearchBuilder(field, vector, k, numCands);
+        if (randomBoolean()) {
+            builder.boost(randomFloat());
+        }
+
+        int numFilters = randomIntBetween(0, 3);
+        for (int i = 0; i < numFilters; i++) {
+            builder.addFilterQuery(QueryBuilders.termQuery(randomAlphaOfLength(5), randomAlphaOfLength(10)));
+        }
+
+        return builder;
+    }
+
     @Before
     public void registerNamedXContents() {
         SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList());
@@ -61,23 +81,7 @@ protected Writeable.Reader<KnnSearchBuilder> instanceReader() {
 
     @Override
     protected KnnSearchBuilder createTestInstance() {
-        String field = randomAlphaOfLength(6);
-        int dim = randomIntBetween(2, 30);
-        float[] vector = randomVector(dim);
-        int k = randomIntBetween(1, 100);
-        int numCands = randomIntBetween(k + 20, 1000);
-
-        KnnSearchBuilder builder = new KnnSearchBuilder(field, vector, k, numCands);
-        if (randomBoolean()) {
-            builder.boost(randomFloat());
-        }
-
-        int numFilters = randomIntBetween(0, 3);
-        for (int i = 0; i < numFilters; i++) {
-            builder.addFilterQuery(QueryBuilders.termQuery(randomAlphaOfLength(5), randomAlphaOfLength(10)));
-        }
-
-        return builder;
+        return randomTestInstance();
     }
 
     @Override