diff --git a/docs/changelog/135604.yaml b/docs/changelog/135604.yaml new file mode 100644 index 0000000000000..d91daf7325c96 --- /dev/null +++ b/docs/changelog/135604.yaml @@ -0,0 +1,5 @@ +pr: 135604 +summary: ES|QL - add `dense_vector` field type +area: ES|QL +type: feature +issues: [] diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/case.md b/docs/reference/query-languages/esql/_snippets/functions/types/case.md index bb20353ebe7c7..dcc18f4f73660 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/case.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/case.md @@ -14,6 +14,8 @@ | boolean | date | | date | | boolean | date_nanos | date_nanos | date_nanos | | boolean | date_nanos | | date_nanos | +| boolean | dense_vector | dense_vector | dense_vector | +| boolean | dense_vector | | dense_vector | | boolean | double | double | double | | boolean | double | | double | | boolean | geo_point | geo_point | geo_point | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/knn.md b/docs/reference/query-languages/esql/_snippets/functions/types/knn.md new file mode 100644 index 0000000000000..8b7a555d81665 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/knn.md @@ -0,0 +1,9 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| field | query | options | result | +| --- | --- | --- | --- | +| dense_vector | dense_vector | named parameters | boolean | +| text | dense_vector | named parameters | boolean | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/to_dense_vector.md b/docs/reference/query-languages/esql/_snippets/functions/types/to_dense_vector.md new file mode 100644 index 0000000000000..6a7d025f6a43d --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/to_dense_vector.md @@ -0,0 +1,11 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| field | result | +| --- | --- | +| double | dense_vector | +| integer | dense_vector | +| keyword | dense_vector | +| long | dense_vector | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md b/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md index 77fd3adaf8c56..1c63eb7b367e8 100644 --- a/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md +++ b/docs/reference/query-languages/esql/_snippets/functions/types/to_string.md @@ -9,6 +9,7 @@ | cartesian_shape | keyword | | date | keyword | | date_nanos | keyword | +| dense_vector | keyword | | double | keyword | | geo_point | keyword | | geo_shape | keyword | diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_cosine.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_cosine.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_cosine.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_dot_product.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_dot_product.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_dot_product.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_hamming.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_hamming.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_hamming.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_l1_norm.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_l1_norm.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_l1_norm.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_l2_norm.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_l2_norm.md new file mode 100644 index 0000000000000..4d9804aae2395 --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_l2_norm.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| left | right | result | +| --- | --- | --- | +| dense_vector | dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/functions/types/v_magnitude.md b/docs/reference/query-languages/esql/_snippets/functions/types/v_magnitude.md new file mode 100644 index 0000000000000..664cb86a5d11e --- /dev/null +++ b/docs/reference/query-languages/esql/_snippets/functions/types/v_magnitude.md @@ -0,0 +1,8 @@ +% This is generated by ESQL's AbstractFunctionTestCase. Do not edit it. See ../README.md for how to regenerate it. + +**Supported types** + +| input | result | +| --- | --- | +| dense_vector | double | + diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md index 0a1179d8b1218..333818f6d05d0 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_not_null.md @@ -12,6 +12,7 @@ | counter_long | boolean | | date | boolean | | date_nanos | boolean | +| dense_vector | boolean | | double | boolean | | geo_point | boolean | | geo_shape | boolean | diff --git a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md index 0a1179d8b1218..333818f6d05d0 100644 --- a/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md +++ b/docs/reference/query-languages/esql/_snippets/operators/types/is_null.md @@ -12,6 +12,7 @@ | counter_long | boolean | | date | boolean | | date_nanos | boolean | +| dense_vector | boolean | | double | boolean | | geo_point | boolean | | geo_shape | boolean | diff --git a/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg b/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg index 54304ee44b11f..36a8f66b3af2a 100644 --- a/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg +++ b/docs/reference/query-languages/esql/images/functions/to_dense_vector.svg @@ -1 +1 @@ -TO_DENSE_VECTOR(field) \ No newline at end of file +TO_DENSE_VECTOR(field) \ No newline at end of file diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/case.json b/docs/reference/query-languages/esql/kibana/definition/functions/case.json index 96a3089e7f63c..b37f20a21dff9 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/case.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/case.json @@ -214,6 +214,48 @@ "variadic" : true, "returnType" : "date_nanos" }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "dense_vector", + "optional" : false, + "description" : "The value that’s returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + } + ], + "variadic" : true, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "condition", + "type" : "boolean", + "optional" : false, + "description" : "A condition." + }, + { + "name" : "trueValue", + "type" : "dense_vector", + "optional" : false, + "description" : "The value that’s returned when the corresponding condition is the first to evaluate to `true`. The default value is returned when no condition matches." + }, + { + "name" : "elseValue", + "type" : "dense_vector", + "optional" : true, + "description" : "The value that’s returned when no condition evaluates to `true`." + } + ], + "variadic" : true, + "returnType" : "dense_vector" + }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json index cd5541f8e7bfe..33845ebd490ce 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/knn.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/knn.json @@ -3,7 +3,58 @@ "type" : "scalar", "name" : "knn", "description" : "Finds the k nearest vectors to a query vector, as measured by a similarity metric. knn function finds nearest vectors through approximate search on indexed dense_vectors or semantic_text fields.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Field that the query will target. knn function can be used with dense_vector or semantic_text fields. Other text fields are not allowed" + }, + { + "name" : "query", + "type" : "dense_vector", + "optional" : false, + "description" : "Vector value to find top nearest neighbours for." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.Defaults to 1.0.'}, {name='min_candidates', values=[10], description='The minimum number of nearest neighbor candidates to consider per shard while doing knn search. KNN may use a higher number of candidates in case the query can't use a approximate results. Cannot exceed 10,000. Increasing min_candidates tends to improve the accuracy of the final results. Defaults to 1.5 * LIMIT used for the query.'}, {name='rescore_oversample', values=[3.5], description='Applies the specified oversampling for rescoring quantized vectors. See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details.'}, {name='similarity', values=[0.01], description='The minimum similarity required for a document to be considered a match. The similarity value calculated relates to the raw similarity used, not the document score.'}", + "optional" : true, + "description" : "(Optional) kNN additional options as <>. See <> for more information." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, + { + "params" : [ + { + "name" : "field", + "type" : "text", + "optional" : false, + "description" : "Field that the query will target. knn function can be used with dense_vector or semantic_text fields. Other text fields are not allowed" + }, + { + "name" : "query", + "type" : "dense_vector", + "optional" : false, + "description" : "Vector value to find top nearest neighbours for." + }, + { + "name" : "options", + "type" : "function_named_parameters", + "mapParams" : "{name='boost', values=[2.5], description='Floating point number used to decrease or increase the relevance scores of the query.Defaults to 1.0.'}, {name='min_candidates', values=[10], description='The minimum number of nearest neighbor candidates to consider per shard while doing knn search. KNN may use a higher number of candidates in case the query can't use a approximate results. Cannot exceed 10,000. Increasing min_candidates tends to improve the accuracy of the final results. Defaults to 1.5 * LIMIT used for the query.'}, {name='rescore_oversample', values=[3.5], description='Applies the specified oversampling for rescoring quantized vectors. See [oversampling and rescoring quantized vectors](docs-content://solutions/search/vector/knn.md#dense-vector-knn-search-rescoring) for details.'}, {name='similarity', values=[0.01], description='The minimum similarity required for a document to be considered a match. The similarity value calculated relates to the raw similarity used, not the document score.'}", + "optional" : true, + "description" : "(Optional) kNN additional options as <>. See <> for more information." + } + ], + "variadic" : false, + "returnType" : "boolean" + } + ], "examples" : [ "from colors metadata _score\n| where knn(rgb_vector, [0, 120, 0])\n| sort _score desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json b/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json index 932937bf10c6c..56949532a872c 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/to_dense_vector.json @@ -3,10 +3,59 @@ "type" : "scalar", "name" : "to_dense_vector", "description" : "Converts a multi-valued input of numbers, or a hexadecimal string, to a dense_vector.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "field", + "type" : "double", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "field", + "type" : "integer", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "field", + "type" : "keyword", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + }, + { + "params" : [ + { + "name" : "field", + "type" : "long", + "optional" : false, + "description" : "multi-valued input of numbers or hexadecimal string to convert." + } + ], + "variadic" : false, + "returnType" : "dense_vector" + } + ], "examples" : [ "row ints = [1, 2, 3]\n| eval vector = to_dense_vector(ints)\n| keep vector" ], - "preview" : false, - "snapshot_only" : true + "preview" : true, + "snapshot_only" : false } diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json b/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json index 7e81f967db12e..e95e4fc682a83 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/to_string.json @@ -64,6 +64,18 @@ "variadic" : false, "returnType" : "keyword" }, + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Input value. The input can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "keyword" + }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json index f3b3df1d88c6a..aac52a2385ce3 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_cosine.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_cosine", "description" : "Calculates the cosine similarity between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate cosine similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate cosine similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | where color != \"black\"\n | eval similarity = v_cosine(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json index 05b8f0aa1c3f7..a00eaf6dc8387 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_dot_product.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_dot_product", "description" : "Calculates the dot product between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate dot product similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate dot product similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_dot_product(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json index 51e3660ae8650..6d9885918f5df 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_hamming.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_hamming", "description" : "Calculates the Hamming distance between two dense vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "First dense_vector to use to calculate the Hamming distance" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "Second dense_vector to use to calculate the Hamming distance" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_hamming(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json index 2dcd2ac95c980..6cdf7ba32aacd 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_l1_norm.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_l1_norm", "description" : "Calculates the l1 norm between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate l1 norm similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate l1 norm similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_l1_norm(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json index 573de7891d893..344d0c1746456 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_l2_norm.json @@ -3,7 +3,26 @@ "type" : "scalar", "name" : "v_l2_norm", "description" : "Calculates the l2 norm between two dense_vectors.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "left", + "type" : "dense_vector", + "optional" : false, + "description" : "first dense_vector to calculate l2 norm similarity" + }, + { + "name" : "right", + "type" : "dense_vector", + "optional" : false, + "description" : "second dense_vector to calculate l2 norm similarity" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval similarity = v_l2_norm(rgb_vector, [0, 255, 255])\n | sort similarity desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json b/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json index 2835d403e656e..1f69905176ed8 100644 --- a/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json +++ b/docs/reference/query-languages/esql/kibana/definition/functions/v_magnitude.json @@ -3,7 +3,20 @@ "type" : "scalar", "name" : "v_magnitude", "description" : "Calculates the magnitude of a dense_vector.", - "signatures" : [ ], + "signatures" : [ + { + "params" : [ + { + "name" : "input", + "type" : "dense_vector", + "optional" : false, + "description" : "dense_vector for which to compute the magnitude" + } + ], + "variadic" : false, + "returnType" : "double" + } + ], "examples" : [ " from colors\n | eval magnitude = v_magnitude(rgb_vector)\n | sort magnitude desc, color asc" ], diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json index c3cc8fae2a14e..2243d4979ba69 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_not_null.json @@ -102,6 +102,18 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Value to check. It can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json index 08c98810ddede..11b7291b34afb 100644 --- a/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json +++ b/docs/reference/query-languages/esql/kibana/definition/operators/is_null.json @@ -102,6 +102,18 @@ "variadic" : false, "returnType" : "boolean" }, + { + "params" : [ + { + "name" : "field", + "type" : "dense_vector", + "optional" : false, + "description" : "Value to check. It can be a single- or multi-valued column or an expression." + } + ], + "variadic" : false, + "returnType" : "boolean" + }, { "params" : [ { diff --git a/server/src/main/resources/transport/definitions/referable/esql_dense_vector_created_version.csv b/server/src/main/resources/transport/definitions/referable/esql_dense_vector_created_version.csv new file mode 100644 index 0000000000000..2752f57900f3e --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_dense_vector_created_version.csv @@ -0,0 +1 @@ +9183000 diff --git a/server/src/main/resources/transport/upper_bounds/9.2.csv b/server/src/main/resources/transport/upper_bounds/9.2.csv index 0dde9b0f74618..f575dcaf4efa8 100644 --- a/server/src/main/resources/transport/upper_bounds/9.2.csv +++ b/server/src/main/resources/transport/upper_bounds/9.2.csv @@ -1 +1 @@ -inference_telemetry_added_semantic_text_stats,9182000 +esql_dense_vector_created_version,9183000 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java index c0ca5d18dc63d..75a6c36a1f052 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/expression/TypeResolutions.java @@ -20,8 +20,10 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.core.type.DataType.isSpatialOrGrid; public final class TypeResolutions { @@ -75,17 +77,27 @@ public static TypeResolution isDate(Expression e, String operationName, ParamOrd /** * @see DataType#isRepresentable(DataType) */ - public static TypeResolution isRepresentableExceptCounters(Expression e, String operationName, ParamOrdinal paramOrd) { - return isType(e, DataType::isRepresentable, operationName, paramOrd, "any type except counter types"); + public static TypeResolution isRepresentableExceptCountersAndDenseVector(Expression e, String operationName, ParamOrdinal paramOrd) { + return isType( + e, + dt -> isRepresentable(dt) && dt != DENSE_VECTOR, + operationName, + paramOrd, + "any type except counter types or dense_vector" + ); } - public static TypeResolution isRepresentableExceptCountersAndSpatial(Expression e, String operationName, ParamOrdinal paramOrd) { + public static TypeResolution isRepresentableExceptCountersSpatialAndDenseVector( + Expression e, + String operationName, + ParamOrdinal paramOrd + ) { return isType( e, - (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t), + (t) -> isSpatialOrGrid(t) == false && DataType.isRepresentable(t) && t != DENSE_VECTOR, operationName, paramOrd, - "any type except counter and spatial types" + "any type except counter, spatial types or dense_vector" ); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java index b9f55f7985e84..4bd722992f524 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/plugin/EsqlCorePlugin.java @@ -14,5 +14,4 @@ public class EsqlCorePlugin extends Plugin implements ExtensiblePlugin { public static final FeatureFlag AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG = new FeatureFlag("esql_aggregate_metric_double"); - public static final FeatureFlag DENSE_VECTOR_FEATURE_FLAG = new FeatureFlag("esql_dense_vector"); } diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 7a7d5e15e5a9e..618f941a0214a 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -35,7 +35,6 @@ import static java.util.stream.Collectors.toMap; import static org.elasticsearch.TransportVersions.INFERENCE_REQUEST_ADAPTIVE_RATE_LIMITING; -import static org.elasticsearch.TransportVersions.ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION; /** * This enum represents data types the ES|QL query processing layer is able to @@ -323,12 +322,7 @@ public enum DataType implements Writeable { * Fields with this type are dense vectors, represented as an array of double values. */ DENSE_VECTOR( - builder().esType("dense_vector") - .estimatedSize(4096) - .createdVersion( - // Version created just *after* we committed support for dense_vector - ML_INFERENCE_SAGEMAKER_CHAT_COMPLETION - ) + builder().esType("dense_vector").estimatedSize(4096).createdVersion(DataTypesTransportVersions.ESQL_DENSE_VECTOR_CREATED_VERSION) ); /** @@ -349,8 +343,7 @@ public enum DataType implements Writeable { * */ public static final Map UNDER_CONSTRUCTION = Map.ofEntries( - Map.entry(AGGREGATE_METRIC_DOUBLE, EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG), - Map.entry(DENSE_VECTOR, EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG) + Map.entry(AGGREGATE_METRIC_DOUBLE, EsqlCorePlugin.AGGREGATE_METRIC_DOUBLE_FEATURE_FLAG) ); private final String typeName; @@ -955,4 +948,10 @@ Builder createdVersion(TransportVersion createdVersion) { return this; } } + + private static class DataTypesTransportVersions { + public static final TransportVersion ESQL_DENSE_VECTOR_CREATED_VERSION = TransportVersion.fromName( + "esql_dense_vector_created_version" + ); + } } diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java index abc35d68a3d36..fa5f139c5c6a1 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/data/BlockUtils.java @@ -9,6 +9,7 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.Randomness; +import org.elasticsearch.common.breaker.CircuitBreakingException; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder.AggregateMetricDoubleLiteral; import org.elasticsearch.core.Releasable; import org.elasticsearch.core.Releasables; @@ -35,19 +36,24 @@ public record BuilderWrapper(Block.Builder builder, Consumer append) imp public BuilderWrapper(Block.Builder builder, Consumer append) { this.builder = builder; this.append = o -> { - if (o == null) { - builder.appendNull(); - return; - } - if (o instanceof List l) { - builder.beginPositionEntry(); - for (Object v : l) { - append.accept(v); + try { + if (o == null) { + builder.appendNull(); + return; + } + if (o instanceof List l) { + builder.beginPositionEntry(); + for (Object v : l) { + append.accept(v); + } + builder.endPositionEntry(); + return; } - builder.endPositionEntry(); - return; + append.accept(o); + } catch (CircuitBreakingException e) { + close(); + throw e; } - append.accept(o); }; } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec index 2db3e96de11c0..5cc4c9bc60882 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-bit.csv-spec @@ -1,5 +1,5 @@ retrieveBitVectorData -required_capability: dense_vector_field_type_bit_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -16,7 +16,7 @@ id:l | bit_vector:dense_vector ; denseBitVectorWithEval -required_capability: dense_vector_field_type_bit_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -33,7 +33,7 @@ id:l | v:dense_vector ; denseBitVectorWithRenameAndDrop -required_capability: dense_vector_field_type_bit_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec index 0ecf23332ad58..e298cc9eda963 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector-byte.csv-spec @@ -1,5 +1,5 @@ retrieveByteVectorData -required_capability: dense_vector_field_type_byte_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -16,7 +16,7 @@ id:l | byte_vector:dense_vector ; denseByteVectorWithEval -required_capability: dense_vector_field_type_byte_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -34,7 +34,7 @@ id:l | v:dense_vector ; denseByteVectorWithRenameAndDrop -required_capability: dense_vector_field_type_byte_elements +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec index b68fd9ba416b6..d0133d409e83e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/dense_vector.csv-spec @@ -1,6 +1,7 @@ retrieveDenseVectorData -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: dense_vector_agg_metric_double_if_fns +required_capability: l2_norm_vector_similarity_function FROM dense_vector | EVAL k = v_l2_norm(float_vector, [1]) // workaround to enable fetching dense_vector @@ -16,7 +17,7 @@ id:l | float_vector:dense_vector ; denseVectorWithEval -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -34,7 +35,7 @@ id:l | v:dense_vector ; denseVectorWithRenameAndDrop -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: l2_norm_vector_similarity_function FROM dense_vector @@ -53,7 +54,7 @@ id:l | new_vector:dense_vector ; convertIntsToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function // tag::to_dense_vector-ints[] @@ -70,7 +71,7 @@ vector:dense_vector ; convertLongsToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function row longs = [5013792, 2147483647, 501379200000] @@ -83,7 +84,7 @@ vector:dense_vector ; convertDoublesToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function row doubles = [123.4, 567.8, 901.2] @@ -96,7 +97,7 @@ vector:dense_vector ; convertHexStringToDenseVector -required_capability: dense_vector_field_type +required_capability: dense_vector_field_type_released required_capability: to_dense_vector_function row hex_str = "0102030405060708090a0b0c0d0e0f" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-cosine-similarity.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-cosine-similarity.csv-spec index 451368deec934..bff56fe2b4af4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-cosine-similarity.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-cosine-similarity.csv-spec @@ -2,6 +2,7 @@ similarityWithVectorField required_capability: cosine_vector_similarity_function + required_capability: dense_vector_field_type_released // tag::vector-cosine-similarity[] from colors @@ -30,6 +31,7 @@ gray | 0.9082483053207397 similarityAsPartOfExpression required_capability: cosine_vector_similarity_function + required_capability: dense_vector_field_type_released from colors | where color != "black" @@ -54,6 +56,7 @@ gray | 1.454 similarityWithLiteralVectors required_capability: cosine_vector_similarity_function +required_capability: dense_vector_field_type_released row a = 1 | eval similarity = round(v_cosine([1, 2, 3], [0, 1, 2]), 3) @@ -66,6 +69,7 @@ similarity:double similarityWithStats required_capability: cosine_vector_similarity_function + required_capability: dense_vector_field_type_released from colors | where color != "black" @@ -80,6 +84,7 @@ avg:double | min:double | max:double similarityWithNull required_capability: cosine_vector_similarity_function required_capability: vector_similarity_functions_support_null +required_capability: dense_vector_field_type_released from colors | eval similarity = v_cosine(rgb_vector, null) @@ -93,6 +98,7 @@ total_null:long similarityWithRow required_capability: cosine_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released row vector = to_dense_vector([1, 2, 3]) | eval similarity = round(v_cosine(vector, [0, 1, 2]), 3) @@ -105,6 +111,7 @@ vector: dense_vector | similarity:double similarityWithVectorField required_capability: cosine_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released from colors | where color != "black" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-dot-product.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-dot-product.csv-spec index 3297ae84db5ff..809c1dd4a16ae 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-dot-product.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-dot-product.csv-spec @@ -2,6 +2,7 @@ similarityWithVectorField required_capability: dot_product_vector_similarity_function + required_capability: dense_vector_field_type_released // tag::vector-dot-product[] from colors @@ -29,6 +30,7 @@ old lace | 60563.0 similarityAsPartOfExpression required_capability: dot_product_vector_similarity_function +required_capability: dense_vector_field_type_released from colors | eval score = round((1 + v_dot_product(rgb_vector, [0, 255, 255]) / 2), 3) @@ -52,6 +54,7 @@ old lace | 30282.5 similarityWithLiteralVectors required_capability: dot_product_vector_similarity_function +required_capability: dense_vector_field_type_released row a = 1 | eval similarity = round(v_dot_product([1, 2, 3], [0, 1, 2]), 3) @@ -64,6 +67,7 @@ similarity:double similarityWithStats required_capability: dot_product_vector_similarity_function +required_capability: dense_vector_field_type_released from colors | eval similarity = round(v_dot_product(rgb_vector, [0, 255, 255]), 3) @@ -77,6 +81,7 @@ avg:double | min:double | max:double similarityWithNull required_capability: dot_product_vector_similarity_function required_capability: vector_similarity_functions_support_null +required_capability: dense_vector_field_type_released from colors | eval similarity = v_dot_product(rgb_vector, null) @@ -91,6 +96,7 @@ total_null:long similarityWithRow required_capability: dot_product_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released row vector = to_dense_vector([1, 2, 3]) | eval similarity = round(v_dot_product(vector, [0, 1, 2]), 3) @@ -103,6 +109,7 @@ vector: dense_vector | similarity:double similarityWithVectorField required_capability: dot_product_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released from colors | eval query = to_dense_vector([0, 255, 255]) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-hamming.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-hamming.csv-spec index 37630c94e62e0..36ce1e05ea7cd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-hamming.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-hamming.csv-spec @@ -2,6 +2,7 @@ similarityWithVectorField required_capability: hamming_vector_similarity_function + required_capability: dense_vector_field_type_released // tag::vector-hamming[] from colors @@ -29,6 +30,7 @@ firebrick | 16.0 similarityAsPartOfExpression required_capability: hamming_vector_similarity_function +required_capability: dense_vector_field_type_released from colors | eval score = round((1 + v_hamming(rgb_vector, [0, 255, 255]) / 2), 3) @@ -52,6 +54,7 @@ firebrick | 9.0 similarityWithLiteralVectors required_capability: hamming_vector_similarity_function +required_capability: dense_vector_field_type_released row a = 1 | eval similarity = round(v_hamming([1, 2, 3], [0, 1, 2]), 3) @@ -64,6 +67,7 @@ similarity:double similarityWithStats required_capability: hamming_vector_similarity_function +required_capability: dense_vector_field_type_released from colors | eval similarity = round(v_hamming(rgb_vector, [0, 255, 255]), 3) @@ -77,6 +81,7 @@ avg:double | min:double | max:double similarityWithNull required_capability: hamming_vector_similarity_function required_capability: vector_similarity_functions_support_null +required_capability: dense_vector_field_type_released from colors | eval similarity = v_hamming(rgb_vector, null) @@ -90,6 +95,7 @@ total_null:long similarityWithRow required_capability: hamming_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released row vector = to_dense_vector([1, 2, 3]) | eval similarity = round(v_hamming(vector, [0, 1, 2]), 3) @@ -102,6 +108,7 @@ vector: dense_vector | similarity:double similarityWithVectorField required_capability: hamming_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released from colors | eval query = to_dense_vector([0, 255, 255]) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l1-norm.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l1-norm.csv-spec index 148d9d0da85a9..8ac7446b62f71 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l1-norm.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l1-norm.csv-spec @@ -2,6 +2,7 @@ similarityWithVectorField required_capability: l1_norm_vector_similarity_function + required_capability: dense_vector_field_type_released // tag::vector-l1-norm[] from colors @@ -29,6 +30,7 @@ gold | 550.0 similarityAsPartOfExpression required_capability: l1_norm_vector_similarity_function +required_capability: dense_vector_field_type_released from colors | eval score = round((1 + v_l1_norm(rgb_vector, [0, 255, 255]) / 2), 3) @@ -52,6 +54,7 @@ gold | 276.0 similarityWithLiteralVectors required_capability: l1_norm_vector_similarity_function +required_capability: dense_vector_field_type_released row a = 1 | eval similarity = round(v_l1_norm([1, 2, 3], [0, 1, 2]), 3) @@ -64,6 +67,7 @@ similarity:double similarityWithStats required_capability: l1_norm_vector_similarity_function +required_capability: dense_vector_field_type_released from colors | eval similarity = round(v_l1_norm(rgb_vector, [0, 255, 255]), 3) @@ -77,6 +81,7 @@ avg:double | min:double | max:double similarityWithNull required_capability: l1_norm_vector_similarity_function required_capability: vector_similarity_functions_support_null +required_capability: dense_vector_field_type_released from colors | eval similarity = v_l1_norm(rgb_vector, null) @@ -90,6 +95,7 @@ total_null:long similarityWithRow required_capability: l1_norm_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released row vector = to_dense_vector([1, 2, 3]) | eval similarity = round(v_l1_norm(vector, [0, 1, 2]), 3) @@ -102,6 +108,7 @@ vector: dense_vector | similarity:double similarityWithVectorField required_capability: l1_norm_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released from colors | eval query = to_dense_vector([0, 255, 255]) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l2-norm.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l2-norm.csv-spec index d150c65e3b2fa..c3295d250dc93 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l2-norm.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-l2-norm.csv-spec @@ -2,6 +2,7 @@ similarityWithVectorField required_capability: l2_norm_vector_similarity_function + required_capability: dense_vector_field_type_released // tag::vector-l2-norm[] from colors @@ -29,6 +30,7 @@ tomato | 351.0227966308594 similarityAsPartOfExpression required_capability: l2_norm_vector_similarity_function + required_capability: dense_vector_field_type_released from colors | eval score = round((1 + v_l2_norm(rgb_vector, [0, 255, 255]) / 2), 3) @@ -52,6 +54,7 @@ tomato | 176.511 similarityWithLiteralVectors required_capability: l2_norm_vector_similarity_function +required_capability: dense_vector_field_type_released row a = 1 | eval similarity = round(v_l2_norm([1, 2, 3], [0, 1, 2]), 3) @@ -64,6 +67,7 @@ similarity:double similarityWithStats required_capability: l2_norm_vector_similarity_function +required_capability: dense_vector_field_type_released from colors | eval similarity = round(v_l2_norm(rgb_vector, [0, 255, 255]), 3) @@ -77,6 +81,7 @@ avg:double | min:double | max:double similarityWithNull required_capability: l2_norm_vector_similarity_function required_capability: vector_similarity_functions_support_null +required_capability: dense_vector_field_type_released from colors | eval similarity = v_l2_norm(rgb_vector, null) @@ -90,6 +95,7 @@ total_null:long similarityWithRow required_capability: l2_norm_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released row vector = to_dense_vector([1, 2, 3]) | eval similarity = round(v_l2_norm(vector, [0, 1, 2]), 3) @@ -102,6 +108,7 @@ vector: dense_vector | similarity:double similarityWithVectorField required_capability: l2_norm_vector_similarity_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released from colors | eval query = to_dense_vector([0, 255, 255]) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-magnitude.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-magnitude.csv-spec index bb6d39735d8e4..91fe3feba2bb8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-magnitude.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/vector-magnitude.csv-spec @@ -2,6 +2,7 @@ magnitudeWithVectorField required_capability: magnitude_scalar_vector_function + required_capability: dense_vector_field_type_released // tag::vector-magnitude[] from colors @@ -29,6 +30,7 @@ linen | 415.93267822265625 magnitudeAsPartOfExpression required_capability: magnitude_scalar_vector_function + required_capability: dense_vector_field_type_released from colors | eval score = round((1 + v_magnitude(rgb_vector) / 2), 3) @@ -52,6 +54,7 @@ linen | 208.966 magnitudeWithLiteralVectors required_capability: magnitude_scalar_vector_function +required_capability: dense_vector_field_type_released row a = 1 | eval magnitude = round(v_magnitude([1, 2, 3]), 3) @@ -64,6 +67,7 @@ magnitude:double magnitudeWithStats required_capability: magnitude_scalar_vector_function + required_capability: dense_vector_field_type_released from colors | eval magnitude = round(v_magnitude(rgb_vector), 3) @@ -76,7 +80,8 @@ avg:double | min:double | max:double magnitudeWithNull required_capability: magnitude_scalar_vector_function - +required_capability: dense_vector_field_type_released + row a = 1 | eval magnitude = v_magnitude(null) | keep magnitude @@ -89,6 +94,7 @@ null magnitudeWithRow required_capability: magnitude_scalar_vector_function required_capability: to_dense_vector_function +required_capability: dense_vector_field_type_released row vector = to_dense_vector([1, 2, 3]) | eval magnitude = round(v_magnitude(vector), 3) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java index 3014e1d4ba48a..93f75f8395ab3 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/DenseVectorFieldTypeIT.java @@ -19,7 +19,6 @@ import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.XContentFactory; import org.elasticsearch.xpack.esql.action.AbstractEsqlIntegTestCase; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.junit.Before; import java.io.IOException; @@ -35,6 +34,7 @@ import static org.elasticsearch.index.IndexSettings.INDEX_MAPPER_SOURCE_MODE_SETTING; import static org.elasticsearch.index.mapper.SourceFieldMapper.Mode.SYNTHETIC; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.L2_NORM_VECTOR_SIMILARITY_FUNCTION; public class DenseVectorFieldTypeIT extends AbstractEsqlIntegTestCase { @@ -91,6 +91,8 @@ public DenseVectorFieldTypeIT( private final Map> indexedVectors = new HashMap<>(); public void testRetrieveFieldType() { + assumeTrue("Need L2_NORM available for dense_vector retrieval", L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()); + var query = """ FROM test | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector @@ -105,6 +107,8 @@ public void testRetrieveFieldType() { @SuppressWarnings("unchecked") public void testRetrieveTopNDenseVectorFieldData() { + assumeTrue("Need L2_NORM available for dense_vector retrieval", L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()); + var query = """ FROM test | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector @@ -133,6 +137,8 @@ public void testRetrieveTopNDenseVectorFieldData() { @SuppressWarnings("unchecked") public void testRetrieveDenseVectorFieldData() { + assumeTrue("Need L2_NORM available for dense_vector retrieval", L2_NORM_VECTOR_SIMILARITY_FUNCTION.isEnabled()); + var query = """ FROM test | EVAL k = v_l2_norm(vector, [1]) // workaround to enable fetching dense_vector @@ -197,8 +203,6 @@ public void testNonIndexedDenseVectorField() throws IOException { @Before public void setup() throws IOException { - assumeTrue("Dense vector type is disabled", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - createIndexWithDenseVector("test"); int numDims = randomIntBetween(32, 64) * 2; // min 64, even number diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java index e886e2baf06c4..f0fc8955f0b39 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/vector/VectorSimilarityFunctionsIT.java @@ -192,8 +192,6 @@ private static float[] readVector(List leftVector) { @Before public void setup() throws IOException { - assumeTrue("Dense vector type is disabled", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); - createIndexWithDenseVector("test"); numDims = randomIntBetween(32, 64) * 2; // min 64, even number diff --git a/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromFloatEvaluator.java b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromFloatEvaluator.java new file mode 100644 index 0000000000000..91b8005e9da4e --- /dev/null +++ b/x-pack/plugin/esql/src/main/generated/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringFromFloatEvaluator.java @@ -0,0 +1,133 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License +// 2.0; you may not use this file except in compliance with the Elastic License +// 2.0. +package org.elasticsearch.xpack.esql.expression.function.scalar.convert; + +import java.lang.Override; +import java.lang.String; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.RamUsageEstimator; +import org.elasticsearch.compute.data.Block; +import org.elasticsearch.compute.data.BytesRefBlock; +import org.elasticsearch.compute.data.FloatBlock; +import org.elasticsearch.compute.data.FloatVector; +import org.elasticsearch.compute.data.Vector; +import org.elasticsearch.compute.operator.DriverContext; +import org.elasticsearch.compute.operator.EvalOperator; +import org.elasticsearch.core.Releasables; +import org.elasticsearch.xpack.esql.core.tree.Source; + +/** + * {@link EvalOperator.ExpressionEvaluator} implementation for {@link ToString}. + * This class is generated. Edit {@code ConvertEvaluatorImplementer} instead. + */ +public final class ToStringFromFloatEvaluator extends AbstractConvertFunction.AbstractEvaluator { + private static final long BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(ToStringFromFloatEvaluator.class); + + private final EvalOperator.ExpressionEvaluator flt; + + public ToStringFromFloatEvaluator(Source source, EvalOperator.ExpressionEvaluator flt, + DriverContext driverContext) { + super(driverContext, source); + this.flt = flt; + } + + @Override + public EvalOperator.ExpressionEvaluator next() { + return flt; + } + + @Override + public Block evalVector(Vector v) { + FloatVector vector = (FloatVector) v; + int positionCount = v.getPositionCount(); + if (vector.isConstant()) { + return driverContext.blockFactory().newConstantBytesRefBlockWith(evalValue(vector, 0), positionCount); + } + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + builder.appendBytesRef(evalValue(vector, p)); + } + return builder.build(); + } + } + + private BytesRef evalValue(FloatVector container, int index) { + float value = container.getFloat(index); + return ToString.fromFloat(value); + } + + @Override + public Block evalBlock(Block b) { + FloatBlock block = (FloatBlock) b; + int positionCount = block.getPositionCount(); + try (BytesRefBlock.Builder builder = driverContext.blockFactory().newBytesRefBlockBuilder(positionCount)) { + for (int p = 0; p < positionCount; p++) { + int valueCount = block.getValueCount(p); + int start = block.getFirstValueIndex(p); + int end = start + valueCount; + boolean positionOpened = false; + boolean valuesAppended = false; + for (int i = start; i < end; i++) { + BytesRef value = evalValue(block, i); + if (positionOpened == false && valueCount > 1) { + builder.beginPositionEntry(); + positionOpened = true; + } + builder.appendBytesRef(value); + valuesAppended = true; + } + if (valuesAppended == false) { + builder.appendNull(); + } else if (positionOpened) { + builder.endPositionEntry(); + } + } + return builder.build(); + } + } + + private BytesRef evalValue(FloatBlock container, int index) { + float value = container.getFloat(index); + return ToString.fromFloat(value); + } + + @Override + public String toString() { + return "ToStringFromFloatEvaluator[" + "flt=" + flt + "]"; + } + + @Override + public void close() { + Releasables.closeExpectNoException(flt); + } + + @Override + public long baseRamBytesUsed() { + long baseRamBytesUsed = BASE_RAM_BYTES_USED; + baseRamBytesUsed += flt.baseRamBytesUsed(); + return baseRamBytesUsed; + } + + public static class Factory implements EvalOperator.ExpressionEvaluator.Factory { + private final Source source; + + private final EvalOperator.ExpressionEvaluator.Factory flt; + + public Factory(Source source, EvalOperator.ExpressionEvaluator.Factory flt) { + this.source = source; + this.flt = flt; + } + + @Override + public ToStringFromFloatEvaluator get(DriverContext context) { + return new ToStringFromFloatEvaluator(source, flt.get(context), context); + } + + @Override + public String toString() { + return "ToStringFromFloatEvaluator[" + "flt=" + flt + "]"; + } + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 818af0aa49ebe..483da630bb4c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -12,7 +12,6 @@ import org.elasticsearch.compute.lucene.read.ValuesSourceReaderOperator; import org.elasticsearch.features.NodeFeature; import org.elasticsearch.rest.action.admin.cluster.RestNodesCapabilitiesAction; -import org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin; import org.elasticsearch.xpack.esql.plugin.EsqlFeatures; import java.util.ArrayList; @@ -1286,7 +1285,7 @@ public enum Cap { /** * Dense vector field type support */ - DENSE_VECTOR_FIELD_TYPE(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG), + DENSE_VECTOR_FIELD_TYPE_RELEASED, /** * Enable support for index aliases in lookup joins @@ -1443,12 +1442,12 @@ public enum Cap { /** * Byte elements dense vector field type support. */ - DENSE_VECTOR_FIELD_TYPE_BYTE_ELEMENTS(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG), + DENSE_VECTOR_FIELD_TYPE_BYTE_ELEMENTS, /** * Bit elements dense vector field type support. */ - DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS(EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG), + DENSE_VECTOR_FIELD_TYPE_BIT_ELEMENTS, /** * Support null elements on vector similarity functions @@ -1516,7 +1515,7 @@ public enum Cap { /** * TO_DENSE_VECTOR function. */ - TO_DENSE_VECTOR_FUNCTION(Build.current().isSnapshot()), + TO_DENSE_VECTOR_FUNCTION, /** * Support present_over_time aggregation that gets evaluated per time-series diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java index 9a8824b16b1d7..ed7e8ebb57003 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/ExpressionWritables.java @@ -10,7 +10,6 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.ExpressionCoreWritables; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.aggregate.AggregateWritables; @@ -216,9 +215,7 @@ public static List unaryScalars() { entries.add(ToDatetime.ENTRY); entries.add(ToDateNanos.ENTRY); entries.add(ToDegrees.ENTRY); - if (EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()) { - entries.add(ToDenseVector.ENTRY); - } + entries.add(ToDenseVector.ENTRY); entries.add(ToDouble.ENTRY); entries.add(ToGeoShape.ENTRY); entries.add(ToCartesianShape.ENTRY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java index 45e7849b6a603..a648c2f77d9c6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/EsqlFunctionRegistry.java @@ -466,6 +466,7 @@ private static FunctionDefinition[][] functions() { def(ToDatetime.class, ToDatetime::new, "to_datetime", "to_dt"), def(ToDateNanos.class, ToDateNanos::new, "to_date_nanos", "to_datenanos"), def(ToDegrees.class, ToDegrees::new, "to_degrees"), + def(ToDenseVector.class, ToDenseVector::new, "to_dense_vector"), def(ToDouble.class, ToDouble::new, "to_double", "to_dbl"), def(ToGeohash.class, ToGeohash::new, "to_geohash"), def(ToGeotile.class, ToGeotile::new, "to_geotile"), @@ -539,7 +540,6 @@ private static FunctionDefinition[][] snapshotFunctions() { def(Last.class, bi(Last::new), "last"), def(Score.class, uni(Score::new), Score.NAME), def(Term.class, bi(Term::new), "term"), - def(ToDenseVector.class, ToDenseVector::new, "to_dense_vector"), def(Knn.class, tri(Knn::new), "knn"), def(CosineSimilarity.class, CosineSimilarity::new, "v_cosine"), def(DotProduct.class, DotProduct::new, "v_dot_product"), diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java index f2661e764a9e0..7337d55b8f7df 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Absent.java @@ -127,7 +127,13 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isCounter() == false, sourceText(), DEFAULT, "any type except counter types"); + return isType( + field(), + dt -> dt.isCounter() == false && dt != DataType.DENSE_VECTOR, + sourceText(), + DEFAULT, + "any type except counter types or dense_vector" + ); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java index 15810e151e623..5ee5f371ca0fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Count.java @@ -144,7 +144,13 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isCounter() == false, sourceText(), DEFAULT, "any type except counter types"); + return isType( + field(), + dt -> dt.isCounter() == false && dt != DataType.DENSE_VECTOR, + sourceText(), + DEFAULT, + "any type except counter types or dense_vector" + ); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java index 345d2cde7ef15..c65601211b544 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Present.java @@ -133,6 +133,12 @@ public Nullability nullable() { @Override protected TypeResolution resolveType() { - return isType(field(), dt -> dt.isCounter() == false, sourceText(), DEFAULT, "any type except counter types"); + return isType( + field(), + dt -> dt.isCounter() == false && dt != DataType.DENSE_VECTOR, + sourceText(), + DEFAULT, + "any type except counter types or dense_vector" + ); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java index 1ed7e439a9f06..66454c7dc6572 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Sample.java @@ -40,7 +40,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isNotNull; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; import static org.elasticsearch.xpack.esql.expression.Foldables.TypeResolutionValidator.forPostOptimizationValidation; import static org.elasticsearch.xpack.esql.expression.Foldables.TypeResolutionValidator.forPreOptimizationValidation; @@ -126,8 +126,9 @@ protected TypeResolution resolveType() { if (childrenResolved() == false) { return new TypeResolution("Unresolved children"); } - var typeResolution = isRepresentableExceptCounters(field(), sourceText(), FIRST).and(isNotNull(limitField(), sourceText(), SECOND)) - .and(isType(limitField(), dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer")); + var typeResolution = isRepresentableExceptCountersAndDenseVector(field(), sourceText(), FIRST).and( + isNotNull(limitField(), sourceText(), SECOND) + ).and(isType(limitField(), dt -> dt == DataType.INTEGER, sourceText(), SECOND, "integer")); if (typeResolution.unresolved()) { return typeResolution; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java index 863a9d2d2d58d..4958bbca017f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/aggregate/Values.java @@ -165,7 +165,7 @@ public DataType dataType() { @Override protected TypeResolution resolveType() { - return TypeResolutions.isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return TypeResolutions.isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java index 3f5cfea2e7443..32198b0f42d17 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/Case.java @@ -68,6 +68,7 @@ ConditionEvaluatorSupplier toEvaluator(ToEvaluator toEvaluator) { "cartesian_shape", "date", "date_nanos", + "dense_vector", "double", "geo_point", "geo_shape", @@ -111,6 +112,7 @@ public Case( "cartesian_shape", "date", "date_nanos", + "dense_vector", "double", "geo_point", "geo_shape", diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java index f70c0a59b2ece..168a08e8b835e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVector.java @@ -49,6 +49,7 @@ public class ToDenseVector extends AbstractConvertFunction { @FunctionInfo( returnType = "dense_vector", description = "Converts a multi-valued input of numbers, or a hexadecimal string, to a dense_vector.", + preview = true, examples = @Example(file = "dense_vector", tag = "to_dense_vector-ints") ) public ToDenseVector( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java index 9cf14aea6b643..9b05e37774bb8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToString.java @@ -31,6 +31,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.CARTESIAN_SHAPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; +import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEOHASH; import static org.elasticsearch.xpack.esql.core.type.DataType.GEOHEX; @@ -62,6 +63,7 @@ public class ToString extends AbstractConvertFunction implements EvaluatorMapper Map.entry(DATETIME, ToStringFromDatetimeEvaluator.Factory::new), Map.entry(DATE_NANOS, ToStringFromDateNanosEvaluator.Factory::new), Map.entry(IP, ToStringFromIPEvaluator.Factory::new), + Map.entry(DENSE_VECTOR, ToStringFromFloatEvaluator.Factory::new), Map.entry(DOUBLE, ToStringFromDoubleEvaluator.Factory::new), Map.entry(LONG, ToStringFromLongEvaluator.Factory::new), Map.entry(INTEGER, ToStringFromIntEvaluator.Factory::new), @@ -96,6 +98,7 @@ public ToString( "cartesian_shape", "date", "date_nanos", + "dense_vector", "double", "geo_point", "geo_shape", @@ -169,6 +172,11 @@ static BytesRef fromDouble(double dbl) { return numericBooleanToString(dbl); } + @ConvertEvaluator(extraName = "FromFloat") + static BytesRef fromFloat(float flt) { + return numericBooleanToString(flt); + } + @ConvertEvaluator(extraName = "FromLong") static BytesRef fromDouble(long lng) { return numericBooleanToString(lng); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java index a950fca954101..6989d9651fd55 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppend.java @@ -40,7 +40,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; /** @@ -148,14 +148,14 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(field1, sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(field1, sourceText(), FIRST); if (resolution.unresolved()) { return resolution; } dataType = field1.dataType().noText(); if (dataType == DataType.NULL) { dataType = field2.dataType().noText(); - return isRepresentableExceptCounters(field2, sourceText(), SECOND); + return isRepresentableExceptCountersAndDenseVector(field2, sourceText(), SECOND); } return isType(field2, t -> t.noText() == dataType, sourceText(), SECOND, dataType.typeName()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java index 052dac0d68b1c..fa4cf12505f71 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContains.java @@ -44,7 +44,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isType; /** @@ -147,12 +147,12 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(left(), sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(left(), sourceText(), FIRST); if (resolution.unresolved()) { return resolution; } if (left().dataType() == DataType.NULL) { - return isRepresentableExceptCounters(right(), sourceText(), SECOND); + return isRepresentableExceptCountersAndDenseVector(right(), sourceText(), SECOND); } return isType(right(), t -> t.noText() == left().dataType().noText(), sourceText(), SECOND, left().dataType().noText().typeName()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java index aa6049207fa05..186149817e6db 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCount.java @@ -26,7 +26,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the count of values. @@ -79,7 +79,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java index c7b7ea0523a45..75e2554b7a106 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupe.java @@ -23,7 +23,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Removes duplicate values from a multivalued field. @@ -95,7 +95,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java index 7afcf0a1641cc..a4b62b5e1dbff 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirst.java @@ -31,7 +31,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the minimum value. @@ -110,7 +110,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java index bbcc6367784d4..a9605434a7950 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLast.java @@ -31,7 +31,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the minimum value. @@ -110,7 +110,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCounters(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java index fc8281188c067..aecbe84c6d009 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMax.java @@ -26,7 +26,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndSpatial; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersSpatialAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the maximum value. @@ -68,7 +68,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCountersAndSpatial(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersSpatialAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java index 36dac6658aa5a..e862090f3ba2e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMin.java @@ -26,7 +26,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.DEFAULT; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndSpatial; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersSpatialAndDenseVector; /** * Reduce a multivalued field to a single valued field containing the minimum value. @@ -68,7 +68,7 @@ public String getWriteableName() { @Override protected TypeResolution resolveFieldType() { - return isRepresentableExceptCountersAndSpatial(field(), sourceText(), DEFAULT); + return isRepresentableExceptCountersSpatialAndDenseVector(field(), sourceText(), DEFAULT); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java index 4ad3426b94719..f025a16c85403 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSlice.java @@ -42,7 +42,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.THIRD; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.type.DataType.INTEGER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.stringToInt; @@ -168,7 +168,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(field, sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(field, sourceText(), FIRST); if (resolution.unresolved()) { return resolution; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java index 8a433bee0a4a1..9731a547dc7d2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSort.java @@ -52,7 +52,7 @@ import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.FIRST; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.ParamOrdinal.SECOND; -import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCounters; +import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isRepresentableExceptCountersAndDenseVector; import static org.elasticsearch.xpack.esql.core.expression.TypeResolutions.isString; import static org.elasticsearch.xpack.esql.expression.Validations.isFoldable; @@ -127,7 +127,7 @@ protected TypeResolution resolveType() { return new TypeResolution("Unresolved children"); } - TypeResolution resolution = isRepresentableExceptCounters(field, sourceText(), FIRST); + TypeResolution resolution = isRepresentableExceptCountersAndDenseVector(field, sourceText(), FIRST); if (resolution.unresolved()) { return resolution; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java index 52efaa79fd846..f9a0eb058acd1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/nulls/IsNotNull.java @@ -56,7 +56,8 @@ public class IsNotNull extends UnaryScalarFunction implements EvaluatorMapper, N "unsigned_long", "counter_long", "counter_integer", - "counter_double" }, + "counter_double", + "dense_vector" }, examples = { @Example(file = "null", tag = "is-not-null") } ) public IsNotNull( @@ -75,7 +76,8 @@ public IsNotNull( "unsigned_long", "counter_long", "counter_integer", - "counter_double" } + "counter_double", + "dense_vector", } ) Expression field ) { super(source, field); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 3997edb8d6872..059cb20b8a08c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -28,7 +28,6 @@ import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.xcontent.XContentBuilder; import org.elasticsearch.xcontent.json.JsonXContent; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.InvalidArgumentException; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -144,6 +143,7 @@ public class EsqlDataTypeConverter { typeToConverter.put(DATETIME, ToDatetime::new); typeToConverter.put(DATE_NANOS, ToDateNanos::new); // ToDegrees, typeless + typeToConverter.put(DENSE_VECTOR, ToDenseVector::new); typeToConverter.put(DOUBLE, ToDouble::new); typeToConverter.put(GEO_POINT, ToGeoPoint::new); typeToConverter.put(GEO_SHAPE, ToGeoShape::new); @@ -159,10 +159,7 @@ public class EsqlDataTypeConverter { typeToConverter.put(VERSION, ToVersion::new); typeToConverter.put(DATE_PERIOD, ToDatePeriod::new); typeToConverter.put(TIME_DURATION, ToTimeDuration::new); - - if (EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()) { - typeToConverter.put(DENSE_VECTOR, ToDenseVector::new); - } + typeToConverter.put(DENSE_VECTOR, ToDenseVector::new); TYPE_TO_CONVERTER_FUNCTION = Collections.unmodifiableMap(typeToConverter); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index a1c348000dc5b..6de798562548d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -135,7 +135,6 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.randomInferenceIdOtherThan; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; -import static org.elasticsearch.xpack.esql.core.plugin.EsqlCorePlugin.DENSE_VECTOR_FEATURE_FLAG; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; @@ -1644,7 +1643,7 @@ public void testRegexOnInt() { public void testUnsupportedTypesWithToString() { // DATE_PERIOD and TIME_DURATION types have been added, but not really patched through the engine; i.e. supported. final String supportedTypes = "aggregate_metric_double or boolean or cartesian_point or cartesian_shape or date_nanos or datetime " - + "or geo_point or geo_shape or geohash or geohex or geotile or ip or numeric or string or version"; + + "or dense_vector or geo_point or geo_shape or geohash or geohex or geotile or ip or numeric or string or version"; verifyUnsupported( "row period = 1 year | eval to_string(period)", "line 1:28: argument of [to_string(period)] must be [" + supportedTypes + "], found value [period] type [date_period]" @@ -2357,9 +2356,7 @@ public void testImplicitCasting() { } public void testDenseVectorImplicitCastingKnn() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkDenseVectorCastingHexKnn("float_vector"); @@ -2436,8 +2433,6 @@ private void checkDenseVectorCastingKnnQueryParams(String fieldName) { } public void testDenseVectorImplicitCastingSimilarityFunctions() { - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { checkDenseVectorImplicitCastingSimilarityFunction( "v_cosine(float_vector, [0.342, 0.164, 0.234])", @@ -2500,8 +2495,6 @@ private void checkDenseVectorImplicitCastingSimilarityFunction(String similarity } public void testDenseVectorEvalCastingSimilarityFunctions() { - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - if (EsqlCapabilities.Cap.COSINE_VECTOR_SIMILARITY_FUNCTION.isEnabled()) { checkDenseVectorEvalCastingSimilarityFunction("v_cosine(float_vector, query)"); checkDenseVectorEvalCastingSimilarityFunction("v_cosine(byte_vector, query)"); @@ -2542,8 +2535,6 @@ private void checkDenseVectorEvalCastingSimilarityFunction(String similarityFunc } public void testVectorFunctionHexImplicitCastingError() { - assumeTrue("dense vector casting must be enabled", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - if (EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()) { checkVectorFunctionHexImplicitCastingError("where knn(float_vector, \"notcorrect\")"); } @@ -3347,7 +3338,7 @@ public void testResolveDenseVector() { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, true)); var plan = analyze("FROM foo", analyzer(resolution, TEST_VERIFIER)); assertThat(plan.output(), hasSize(1)); - assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR_FEATURE_FLAG.isEnabled() ? DENSE_VECTOR : UNSUPPORTED)); + assertThat(plan.output().getFirst().dataType(), equalTo(DENSE_VECTOR)); } { IndexResolution resolution = IndexResolver.mergedMappings("foo", new IndexResolver.FieldsInfo(caps, true, false)); @@ -4157,9 +4148,7 @@ public void testRerankInvalidQueryTypes() { } public void testRerankFieldsInvalidTypes() { - List invalidFieldNames = DENSE_VECTOR_FEATURE_FLAG.isEnabled() - ? List.of("date", "date_nanos", "ip", "version", "dense_vector") - : List.of("date", "date_nanos", "ip", "version"); + List invalidFieldNames = List.of("date", "date_nanos", "ip", "version", "dense_vector"); for (String fieldName : invalidFieldNames) { LogManager.getLogger(AnalyzerTests.class).warn("[{}]", fieldName); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index 95e2e15da7d91..305d495b21e62 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -1135,7 +1135,7 @@ public void testAggregateOnCounter() { assertThat( error("FROM tests | STATS count(network.bytes_out)", tsdb), equalTo( - "1:20: argument of [count(network.bytes_out)] must be [any type except counter types]," + "1:20: argument of [count(network.bytes_out)] must be [any type except counter types or dense_vector]," + " found value [network.bytes_out] type [counter_long]" ) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java index 930bfc7cf7203..5107c55503e8a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/TestCaseSupplier.java @@ -53,6 +53,7 @@ import java.util.function.UnaryOperator; import java.util.stream.Collectors; +import static org.elasticsearch.test.ESTestCase.randomFloatBetween; import static org.elasticsearch.test.ESTestCase.randomIntBetween; import static org.elasticsearch.xpack.esql.core.util.NumericUtils.UNSIGNED_LONG_MAX; import static org.elasticsearch.xpack.esql.core.util.SpatialCoordinateTypes.CARTESIAN; @@ -647,6 +648,33 @@ public static void forUnaryDouble( ); } + /** + * Generate positive test cases for a unary function operating on a {@link DataType#DENSE_VECTOR}. + */ + @SuppressWarnings("unchecked") + public static void forUnaryDenseVector( + List suppliers, + String expectedEvaluatorToString, + DataType expectedType, + Function, Object> expectedValue, + float lowerBound, + float upperBound + ) { + List cases = new ArrayList<>(); + cases.add(new TypedDataSupplier("", () -> randomDenseVector(lowerBound, upperBound), DataType.DENSE_VECTOR)); + + unary(suppliers, expectedEvaluatorToString, cases, expectedType, v -> expectedValue.apply((List) v), List.of()); + } + + private static List randomDenseVector(float lower, float upper) { + int dimensions = randomIntBetween(64, 128); + List vector = new ArrayList<>(); + for (int i = 0; i < dimensions; i++) { + vector.add(randomFloatBetween(lower, upper, true)); + } + return vector; + } + /** * Generate positive test cases for a unary function operating on an {@link DataType#BOOLEAN}. */ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java index 42920212a61bd..0e19d0c49953d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/AbsentErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java index e634994572680..c66c4aa9aaeec 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/CountErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java index b394fa803f7ce..0be345a9799e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/PresentErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java index a343faa50f1ad..2b6c955020312 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/SampleErrorTests.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class SampleErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -32,6 +33,17 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { + if (signature.getFirst() == DataType.DENSE_VECTOR) { + return equalTo( + "first argument of [" + + sourceForSignature(signature) + + "] must be [any type except counter types" + + " or dense_vector], found value [] type [dense_vector]" + ); + } + if (signature.getFirst() == DataType.NULL && signature.get(1) == DataType.DENSE_VECTOR) { + return containsString(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "integer")); + } if (signature.get(1).equals(DataType.NULL)) { return equalTo("second argument of [" + sourceForSignature(signature) + "] cannot be null, received []"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java index 1467aa129b467..74afcd0482407 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/aggregate/ValuesErrorTests.java @@ -32,12 +32,13 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - assert false : "All checked types must work"; - return null; + return equalTo( + "argument of [dense_vector] must be [any type except counter types or dense_vector], found value [] type [dense_vector]" + ); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - assertThat(checked, equalTo(0)); + assertThat(checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java index 49a6d3c904203..9871b9ce2e25e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/fulltext/KnnTests.java @@ -35,6 +35,7 @@ import static org.elasticsearch.xpack.esql.SerializationTestUtils.serializeDeserialize; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DENSE_VECTOR; +import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.planner.TranslatorHandler.TRANSLATOR_HANDLER; import static org.hamcrest.Matchers.equalTo; @@ -60,7 +61,7 @@ private static List testCaseSuppliers() { suppliers.add( new TestCaseSupplier( - List.of(DENSE_VECTOR, DENSE_VECTOR, DataType.INTEGER), + List.of(DENSE_VECTOR, DENSE_VECTOR), () -> new TestCaseSupplier.TestCase( List.of( new TestCaseSupplier.TypedData( @@ -72,8 +73,29 @@ private static List testCaseSuppliers() { DENSE_VECTOR, "dense_vector field" ), - new TestCaseSupplier.TypedData(randomDenseVector(), DENSE_VECTOR, "query"), - new TestCaseSupplier.TypedData(randomIntBetween(1, 1000), DataType.INTEGER, "k") + new TestCaseSupplier.TypedData(randomDenseVector(), DENSE_VECTOR, "query") + ), + equalTo("KnnEvaluator" + KnnTests.class.getSimpleName()), + BOOLEAN, + equalTo(true) + ) + ) + ); + suppliers.add( + new TestCaseSupplier( + List.of(TEXT, DENSE_VECTOR), + () -> new TestCaseSupplier.TestCase( + List.of( + new TestCaseSupplier.TypedData( + new FieldAttribute( + Source.EMPTY, + randomIdentifier(), + new EsField(randomIdentifier(), TEXT, Map.of(), false, EsField.TimeSeriesFieldType.NONE) + ), + TEXT, + "text field" + ), + new TestCaseSupplier.TypedData(randomDenseVector(), DENSE_VECTOR, "query") ), equalTo("KnnEvaluator" + KnnTests.class.getSimpleName()), BOOLEAN, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java index a538f17969bce..400dd7d09e841 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/conditional/CaseTests.java @@ -46,6 +46,7 @@ public class CaseTests extends AbstractScalarFunctionTestCase { DataType.BOOLEAN, DataType.DATETIME, DataType.DATE_NANOS, + DataType.DENSE_VECTOR, DataType.DOUBLE, DataType.INTEGER, DataType.LONG, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java index e4e153d25bf8f..19b1b8bf264ae 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToDenseVectorTests.java @@ -12,13 +12,11 @@ import org.apache.lucene.util.BytesRef; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase; import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier; -import org.junit.BeforeClass; import java.util.ArrayList; import java.util.Arrays; @@ -31,11 +29,6 @@ public class ToDenseVectorTests extends AbstractScalarFunctionTestCase { - @BeforeClass - public static void checkCapability() { - assumeTrue("To_DenseVector function capability", EsqlCapabilities.Cap.TO_DENSE_VECTOR_FUNCTION.isEnabled()); - } - public ToDenseVectorTests(@Name("TestCase") Supplier testCaseSupplier) { this.testCase = testCaseSupplier.get(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java index 8d8f59b78114d..dac09579611b9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/convert/ToStringTests.java @@ -35,6 +35,7 @@ public ToStringTests(@Name("TestCase") Supplier testC this.testCase = testCaseSupplier.get(); } + @SuppressWarnings("unchecked") @ParametersFactory public static Iterable parameters() { // TODO multivalue fields @@ -76,6 +77,14 @@ public static Iterable parameters() { Double.POSITIVE_INFINITY, List.of() ); + TestCaseSupplier.forUnaryDenseVector( + suppliers, + "ToStringFromFloatEvaluator[flt=" + read + "]", + DataType.KEYWORD, + d -> d.stream().map(f -> new BytesRef(f.toString())).toList(), + -1.0f, + 1.0f + ); TestCaseSupplier.forUnaryBoolean( suppliers, "ToStringFromBooleanEvaluator[bool=" + read + "]", diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java index df9ab4764c879..7cddacf1aec9c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvAppendErrorTests.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class MvAppendErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -32,14 +33,21 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo( - "second argument of [" - + sourceForSignature(signature) - + "] must be [" - + signature.get(0).noText().typeName() - + "], found value [] type [" - + signature.get(1).typeName() - + "]" - ); + if (signature.getFirst() == DataType.DENSE_VECTOR + || signature.getFirst() == DataType.NULL && signature.get(1) == DataType.DENSE_VECTOR) { + return containsString( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector") + ); + } else { + return equalTo( + "second argument of [" + + sourceForSignature(signature) + + "] must be [" + + signature.get(0).noText().typeName() + + "], found value [] type [" + + signature.get(1).typeName() + + "]" + ); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java index c64cb8eef8d6f..0549a72a10131 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvContainsErrorTests.java @@ -17,6 +17,7 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; public class MvContainsErrorTests extends ErrorsForCasesWithoutExamplesTestCase { @@ -32,14 +33,21 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo( - "second argument of [" - + sourceForSignature(signature) - + "] must be [" - + signature.get(0).noText().typeName() - + "], found value [] type [" - + signature.get(1).typeName() - + "]" - ); + if (signature.getFirst() == DataType.DENSE_VECTOR + || signature.getFirst() == DataType.NULL && signature.get(1) == DataType.DENSE_VECTOR) { + return containsString( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector") + ); + } else { + return equalTo( + "second argument of [" + + sourceForSignature(signature) + + "] must be [" + + signature.get(0).noText().typeName() + + "], found value [] type [" + + signature.get(1).typeName() + + "]" + ); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java index d59a1aa2eb098..0634781d5492f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvCountErrorTests.java @@ -32,22 +32,12 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvCount should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - /* - * In general MvCount should support all signatures. While building a - * new type you may we to temporarily relax this. - */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java index 55f34d9a72f41..d16347efbfd77 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvDedupeErrorTests.java @@ -32,13 +32,7 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvDedupe should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override @@ -47,7 +41,7 @@ protected void assertNumberOfCheckedSignatures(int checked) { * In general MvDedupe should support all signatures. While building a * new type you may we to temporarily relax this. */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java index 7ca829a7629c5..8b296ae5f32b4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvFirstErrorTests.java @@ -32,22 +32,11 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvFirst should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - /* - * In general MvFirst should support all signatures. While building a - * new type you may we to temporarily relax this. - */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } - } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java index 3db13f0368a88..9e5e6de7b7534 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvLastErrorTests.java @@ -32,21 +32,11 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> { - /* - * In general MvLast should support all signatures. While building a - * new type you may we to temporarily remove this. - */ - throw new UnsupportedOperationException("all signatures should be supported"); - })); + return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter types or dense_vector")); } @Override protected void assertNumberOfCheckedSignatures(int checked) { - /* - * In general MvLast should support all signatures. While building a - * new type you may we to temporarily relax this. - */ - assertThat("all signatures should be supported", checked, equalTo(0)); + assertThat("all signatures except dense_vector should be supported", checked, equalTo(1)); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java index 6eee825de85fe..e020bd8b6562c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMaxErrorTests.java @@ -32,6 +32,8 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter and spatial types")); + return equalTo( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter, spatial types or dense_vector") + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java index b9d97d7bb4cb8..4751abcd00891 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvMinErrorTests.java @@ -32,6 +32,8 @@ protected Expression build(Source source, List args) { @Override protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { - return equalTo(typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter and spatial types")); + return equalTo( + typeErrorMessage(false, validPerPosition, signature, (v, p) -> "any type except counter, spatial types or dense_vector") + ); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java index 83d0e4fcf3d75..d0daf48fd9a12 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/multivalue/MvSliceErrorTests.java @@ -34,7 +34,7 @@ protected Expression build(Source source, List args) { protected Matcher expectedTypeErrorMatcher(List> validPerPosition, List signature) { return equalTo(typeErrorMessage(true, validPerPosition, signature, (v, p) -> switch (p) { case 1, 2 -> "integer"; - default -> throw new UnsupportedOperationException(); + default -> "any type except counter types or dense_vector"; })); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java index cecb6a2987fd4..16eae27a35d6b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EqualsErrorTests.java @@ -36,6 +36,6 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP } private static final String TYPE_ERROR = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long," - + " semantic_text, text, unsigned_long or version"; + "boolean, cartesian_point, cartesian_shape, date_nanos, datetime, double, geo_point, geo_shape, geohash, geohex, geotile, " + + "integer, ip, keyword, long, text, unsigned_long or version"; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java index ed0a477ec613c..a748948783e5d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/NotEqualsErrorTests.java @@ -36,6 +36,6 @@ protected Matcher expectedTypeErrorMatcher(List> validPerP } private static final String TYPE_ERROR = - "boolean, cartesian_point, cartesian_shape, datetime, date_nanos, double, geo_point, geo_shape, integer, ip, keyword, long, text, " - + "unsigned_long or version"; + "boolean, cartesian_point, cartesian_shape, date_nanos, datetime, double, geo_point, geo_shape, geohash, geohex, geotile, " + + "integer, ip, keyword, long, text, unsigned_long or version"; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index 3a915bb937ce4..d53c370a572d5 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -1384,7 +1384,6 @@ public void testMultiMatchOptionsPushDown() { } public void testKnnOptionsPushDown() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ @@ -1411,7 +1410,6 @@ public void testKnnOptionsPushDown() { } public void testKnnUsesLimitForK() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ @@ -1430,7 +1428,6 @@ public void testKnnUsesLimitForK() { } public void testKnnKAndMinCandidatesLowerK() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ @@ -1449,7 +1446,6 @@ public void testKnnKAndMinCandidatesLowerK() { } public void testKnnKAndMinCandidatesHigherK() { - assumeTrue("dense_vector capability not available", EsqlCapabilities.Cap.DENSE_VECTOR_FIELD_TYPE.isEnabled()); assumeTrue("knn capability not available", EsqlCapabilities.Cap.KNN_FUNCTION_V5.isEnabled()); String query = """ diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/230_folding.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/230_folding.yml index 5bf382d67578f..da1c9e1ba0fd4 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/230_folding.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/230_folding.yml @@ -5,7 +5,7 @@ setup: - method: POST path: /_query parameters: [ ] - capabilities: [ dense_vector_field_type ] + capabilities: [ dense_vector_field_type_released ] test_runner_features: [ capabilities, contains ] reason: "make sure new functions run where supported only, dense vector needed for index creation" - do: diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index 5b9249b262c7d..453d86a1bdf35 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -145,7 +145,7 @@ unsupported: - method: POST path: /_query parameters: [] - capabilities: [dense_vector_field_type, dense_vector_agg_metric_double_if_fns] + capabilities: [dense_vector_field_type_released, dense_vector_agg_metric_double_if_fns] reason: "uses original_type" - do: @@ -338,7 +338,7 @@ unsupported with sort: - method: POST path: /_query parameters: [ ] - capabilities: [ dense_vector_field_type, dense_vector_agg_metric_double_if_fns ] + capabilities: [ dense_vector_field_type_released, dense_vector_agg_metric_double_if_fns ] reason: "support for sorting when dense_vector_field_type present" - do: