Skip to content

Commit 6575af9

Browse files
authored
[8.19] ES|QL - Allow full text functions to be used in STATS ... WHERE (#125479) (#127981)
* Non existing synonyms sets do not fail shard recovery (#125659) (cherry picked from commit 968bddc) # Conflicts: # qa/mixed-cluster/build.gradle # rest-api-spec/build.gradle * Fix backport merge
1 parent f0bf9ea commit 6575af9

File tree

20 files changed

+492
-34
lines changed

20 files changed

+492
-34
lines changed

docs/changelog/125479.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 125479
2+
summary: ES|QL - Allow full text functions to be used in STATS
3+
area: ES|QL
4+
type: enhancement
5+
issues:
6+
- 125481

x-pack/plugin/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,7 @@ tasks.named("yamlRestTestV7CompatTransform").configure({ task ->
211211
task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors")
212212
task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields")
213213
task.skipTest("esql/180_match_operator/match with functions", "Error message changed")
214+
task.skipTest("esql/180_match_operator/match within eval", "Error message changed")
214215
task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed")
215216
task.skipTest("esql/190_lookup_join/Alias as lookup index", "LOOKUP JOIN does not support index aliases for now")
216217
task.skipTest("esql/190_lookup_join/alias-repeated-alias", "LOOKUP JOIN does not support index aliases for now")

x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -191,3 +191,82 @@ book_no:keyword
191191
7140
192192
2714
193193
;
194+
195+
testKqlInStatsNonPushable
196+
required_capability: kql_function
197+
required_capability: full_text_functions_in_stats_where
198+
199+
from books
200+
| where length(title) > 40
201+
| stats c = count(*) where kql("title:Lord")
202+
;
203+
204+
c:long
205+
3
206+
;
207+
208+
209+
testMatchInStatsPushableAndNonPushable
210+
required_capability: kql_function
211+
required_capability: full_text_functions_in_stats_where
212+
213+
from books
214+
| stats c = count(*) where (kql("title: lord") and ratings > 4.5) or (kql("author: dostoevsky") and length(title) > 50)
215+
;
216+
217+
c:long
218+
6
219+
;
220+
221+
testKqlInStatsPushable
222+
required_capability: kql_function
223+
required_capability: full_text_functions_in_stats_where
224+
225+
from books
226+
| stats c = count(*) where kql("author:tolkien")
227+
;
228+
229+
c:long
230+
22
231+
;
232+
233+
testKqlInStatsWithNonPushableDisjunctions
234+
required_capability: kql_function
235+
required_capability: full_text_functions_in_stats_where
236+
FROM books
237+
| STATS c = count(*) where kql("title: lord") or length(title) > 130
238+
;
239+
240+
c:long
241+
5
242+
;
243+
244+
testKqlInStatsWithMultipleAggs
245+
required_capability: kql_function
246+
required_capability: full_text_functions_in_stats_where
247+
FROM books
248+
| STATS c = count(*) where kql("title: lord"), m = max(book_no::integer) where kql("author: tolkien"), n = min(book_no::integer) where kql("author: dostoevsky")
249+
;
250+
251+
c:long | m:integer | n:integer
252+
4 | 9607 | 1211
253+
;
254+
255+
256+
testKqlInStatsWithGrouping
257+
required_capability: kql_function
258+
required_capability: full_text_functions_in_stats_where
259+
FROM books
260+
| STATS r = AVG(ratings) where kql("title: Lord AND Rings") by author | WHERE r is not null
261+
;
262+
ignoreOrder: true
263+
264+
r:double | author: text
265+
4.75 | Alan Lee
266+
4.674999952316284 | J. R. R. Tolkien
267+
4.670000076293945 | John Ronald Reuel Tolkien
268+
4.670000076293945 | Agnes Perkins
269+
4.670000076293945 | Charles Adolph Huttar
270+
4.670000076293945 | Walter Scheps
271+
4.559999942779541 | J.R.R. Tolkien
272+
;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,3 +755,94 @@ book_no:keyword
755755
7140
756756
2714
757757
;
758+
759+
testMatchInStatsNonPushable
760+
required_capability: match_function
761+
required_capability: full_text_functions_in_stats_where
762+
763+
from books
764+
| where length(title) > 40
765+
| stats c = count(*) where match(title, "Lord")
766+
;
767+
768+
c:long
769+
3
770+
;
771+
772+
testMatchInStatsPushableAndNonPushable
773+
required_capability: match_function
774+
required_capability: full_text_functions_in_stats_where
775+
776+
from books
777+
| stats c = count(*) where (match(title, "lord") and ratings > 4.5) or (match(author, "dostoevsky") and length(title) > 50)
778+
;
779+
780+
c:long
781+
6
782+
;
783+
784+
testMatchInStatsPushable
785+
required_capability: match_function
786+
required_capability: full_text_functions_in_stats_where
787+
788+
from books
789+
| stats c = count(*) where match(author, "tolkien")
790+
;
791+
792+
c:long
793+
22
794+
;
795+
796+
testMatchInStatsWithOptions
797+
required_capability: match_function
798+
required_capability: full_text_functions_in_stats_where
799+
800+
FROM books
801+
| STATS c = count(*) where match(title, "Hobbit Back Again", {"operator": "AND"})
802+
;
803+
804+
c:long
805+
1
806+
;
807+
808+
testMatchInStatsWithNonPushableDisjunctions
809+
required_capability: match_function
810+
required_capability: full_text_functions_in_stats_where
811+
812+
FROM books
813+
| STATS c = count(*) where match(title, "lord") or length(title) > 130
814+
;
815+
816+
c:long
817+
5
818+
;
819+
820+
testMatchInStatsWithMultipleAggs
821+
required_capability: match_function
822+
required_capability: full_text_functions_in_stats_where
823+
FROM books
824+
| STATS c = count(*) where match(title, "lord"), m = max(book_no::integer) where match(author, "tolkien"), n = min(book_no::integer) where match(author, "dostoevsky")
825+
;
826+
827+
c:long | m:integer | n:integer
828+
4 | 9607 | 1211
829+
;
830+
831+
832+
testMatchInStatsWithGrouping
833+
required_capability: match_function
834+
required_capability: full_text_functions_in_stats_where
835+
FROM books
836+
| STATS r = AVG(ratings) where match(title, "Lord Rings", {"operator": "AND"}) by author | WHERE r is not null
837+
;
838+
ignoreOrder: true
839+
840+
r:double | author: text
841+
4.75 | Alan Lee
842+
4.674999952316284 | J. R. R. Tolkien
843+
4.670000076293945 | John Ronald Reuel Tolkien
844+
4.670000076293945 | Agnes Perkins
845+
4.670000076293945 | Charles Adolph Huttar
846+
4.670000076293945 | Walter Scheps
847+
4.559999942779541 | J.R.R. Tolkien
848+
;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -757,3 +757,29 @@ from semantic_text
757757
host:keyword | semantic_text_field:text | language_name:keyword | language_code:integer
758758
"host1" | live long and prosper | English | 1
759759
;
760+
761+
762+
testMatchInStatsNonPushable
763+
required_capability: match_operator_colon
764+
required_capability: full_text_functions_in_stats_where
765+
766+
from books
767+
| where length(title) > 40
768+
| stats c = count(*) where title:"Lord"
769+
;
770+
771+
c:long
772+
3
773+
;
774+
775+
testMatchInStatsPushable
776+
required_capability: match_operator_colon
777+
required_capability: full_text_functions_in_stats_where
778+
779+
from books
780+
| stats c = count(*) where author:"tolkien"
781+
;
782+
783+
c:long
784+
22
785+
;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,3 +211,92 @@ book_no:keyword | title:text
211211
7480 | The Hobbit
212212
// end::qstr-with-options-result[]
213213
;
214+
215+
testQstrInStatsNonPushable
216+
required_capability: qstr_function
217+
required_capability: full_text_functions_in_stats_where
218+
219+
from books
220+
| where length(title) > 40
221+
| stats c = count(*) where qstr("title:Lord")
222+
;
223+
224+
c:long
225+
3
226+
;
227+
228+
testMatchInStatsPushableAndNonPushable
229+
required_capability: qstr_function
230+
required_capability: full_text_functions_in_stats_where
231+
232+
from books
233+
| stats c = count(*) where (qstr("title: lord") and ratings > 4.5) or (qstr("author: dostoevsky") and length(title) > 50)
234+
;
235+
236+
c:long
237+
6
238+
;
239+
240+
testQstrInStatsPushable
241+
required_capability: qstr_function
242+
required_capability: full_text_functions_in_stats_where
243+
244+
from books
245+
| stats c = count(*) where qstr("author:tolkien")
246+
;
247+
248+
c:long
249+
22
250+
;
251+
252+
testQstrInStatsWithOptions
253+
required_capability: qstr_function
254+
required_capability: full_text_functions_in_stats_where
255+
256+
FROM books
257+
| STATS c = count(*) where qstr("title: Hobbit Back Again", {"default_operator": "AND"})
258+
;
259+
260+
c:long
261+
1
262+
;
263+
264+
testQstrInStatsWithNonPushableDisjunctions
265+
required_capability: qstr_function
266+
required_capability: full_text_functions_in_stats_where
267+
FROM books
268+
| STATS c = count(*) where qstr("title: lord") or length(title) > 130
269+
;
270+
271+
c:long
272+
5
273+
;
274+
275+
testQstrInStatsWithMultipleAggs
276+
required_capability: qstr_function
277+
required_capability: full_text_functions_in_stats_where
278+
FROM books
279+
| STATS c = count(*) where qstr("title: lord"), m = max(book_no::integer) where qstr("author: tolkien"), n = min(book_no::integer) where qstr("author: dostoevsky")
280+
;
281+
282+
c:long | m:integer | n:integer
283+
4 | 9607 | 1211
284+
;
285+
286+
testQstrInStatsWithGrouping
287+
required_capability: qstr_function
288+
required_capability: full_text_functions_in_stats_where
289+
FROM books
290+
| STATS r = AVG(ratings) where qstr("title: Lord Rings", {"default_operator": "AND"}) by author | WHERE r is not null
291+
;
292+
ignoreOrder: true
293+
294+
r:double | author: text
295+
4.75 | Alan Lee
296+
4.674999952316284 | J. R. R. Tolkien
297+
4.670000076293945 | John Ronald Reuel Tolkien
298+
4.670000076293945 | Agnes Perkins
299+
4.670000076293945 | Charles Adolph Huttar
300+
4.670000076293945 | Walter Scheps
301+
4.559999942779541 | J.R.R. Tolkien
302+
;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -454,7 +454,6 @@ book_no:keyword | _score:double
454454
8678 | 0.0
455455
;
456456

457-
458457
disjunctionScoresMultipleClauses
459458

460459
required_capability: metadata_score
@@ -477,3 +476,18 @@ book_no:keyword | _score:double
477476
4023 | 1.5062403678894043
478477
2924 | 1.2732219696044922
479478
;
479+
480+
statsScores
481+
482+
required_capability: metadata_score
483+
required_capability: match_function
484+
required_capability: full_text_functions_in_stats_where
485+
486+
from books metadata _score
487+
| where match(title, "Lord Rings", {"operator": "AND"})
488+
| stats avg_score = avg(_score), max_score = max(_score), min_score = min(_score)
489+
;
490+
491+
avg_score:double | max_score:double | min_score:double
492+
3.869828939437866 | 5.123856544494629 | 3.0124807357788086
493+
;

x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void testKqlQueryWithinEval() {
6666
""";
6767

6868
var error = expectThrows(VerificationException.class, () -> run(query));
69-
assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE commands"));
69+
assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE and STATS commands"));
7070
}
7171

7272
public void testInvalidKqlQueryEof() {

0 commit comments

Comments
 (0)