Skip to content

Commit 7ff25b5

Browse files
carlosdelestcarlosdelest
authored
ES|QL - Allow full text functions to be used in STATS ... WHERE (#125479)
1 parent 14a7350 commit 7ff25b5

File tree

20 files changed

+490
-31
lines changed

20 files changed

+490
-31
lines changed

‎docs/changelog/125479.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 125479
2+
summary: ES|QL - Allow full text functions to be used in STATS
3+
area: ES|QL
4+
type: enhancement
5+
issues:
6+
- 125481

‎x-pack/plugin/build.gradle

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,7 @@ tasks.named("yamlRestCompatTestTransform").configure({ task ->
9595
task.skipTest("esql/61_enrich_ip/Invalid IP strings", "We switched from exceptions to null+warnings for ENRICH runtime errors")
9696
task.skipTest("esql/180_match_operator/match with non text field", "Match operator can now be used on non-text fields")
9797
task.skipTest("esql/180_match_operator/match with functions", "Error message changed")
98+
task.skipTest("esql/180_match_operator/match within eval", "Error message changed")
9899
task.skipTest("esql/40_unsupported_types/semantic_text declared in mapping", "The semantic text field format changed")
99100
task.skipTest("esql/190_lookup_join/Alias as lookup index", "LOOKUP JOIN does not support index aliases for now")
100101
task.skipTest("esql/190_lookup_join/alias-repeated-alias", "LOOKUP JOIN does not support index aliases for now")

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/kql-function.csv-spec

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,3 +189,82 @@ book_no:keyword
189189
7140
190190
2714
191191
;
192+
193+
testKqlInStatsNonPushable
194+
required_capability: kql_function
195+
required_capability: full_text_functions_in_stats_where
196+
197+
from books
198+
| where length(title) > 40
199+
| stats c = count(*) where kql("title:Lord")
200+
;
201+
202+
c:long
203+
3
204+
;
205+
206+
207+
testMatchInStatsPushableAndNonPushable
208+
required_capability: kql_function
209+
required_capability: full_text_functions_in_stats_where
210+
211+
from books
212+
| stats c = count(*) where (kql("title: lord") and ratings > 4.5) or (kql("author: dostoevsky") and length(title) > 50)
213+
;
214+
215+
c:long
216+
6
217+
;
218+
219+
testKqlInStatsPushable
220+
required_capability: kql_function
221+
required_capability: full_text_functions_in_stats_where
222+
223+
from books
224+
| stats c = count(*) where kql("author:tolkien")
225+
;
226+
227+
c:long
228+
22
229+
;
230+
231+
testKqlInStatsWithNonPushableDisjunctions
232+
required_capability: kql_function
233+
required_capability: full_text_functions_in_stats_where
234+
FROM books
235+
| STATS c = count(*) where kql("title: lord") or length(title) > 130
236+
;
237+
238+
c:long
239+
5
240+
;
241+
242+
testKqlInStatsWithMultipleAggs
243+
required_capability: kql_function
244+
required_capability: full_text_functions_in_stats_where
245+
FROM books
246+
| STATS c = count(*) where kql("title: lord"), m = max(book_no::integer) where kql("author: tolkien"), n = min(book_no::integer) where kql("author: dostoevsky")
247+
;
248+
249+
c:long | m:integer | n:integer
250+
4 | 9607 | 1211
251+
;
252+
253+
254+
testKqlInStatsWithGrouping
255+
required_capability: kql_function
256+
required_capability: full_text_functions_in_stats_where
257+
FROM books
258+
| STATS r = AVG(ratings) where kql("title: Lord AND Rings") by author | WHERE r is not null
259+
;
260+
ignoreOrder: true
261+
262+
r:double | author: text
263+
4.75 | Alan Lee
264+
4.674999952316284 | J. R. R. Tolkien
265+
4.670000076293945 | John Ronald Reuel Tolkien
266+
4.670000076293945 | Agnes Perkins
267+
4.670000076293945 | Charles Adolph Huttar
268+
4.670000076293945 | Walter Scheps
269+
4.559999942779541 | J.R.R. Tolkien
270+
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-function.csv-spec

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -750,3 +750,94 @@ book_no:keyword
750750
7140
751751
2714
752752
;
753+
754+
testMatchInStatsNonPushable
755+
required_capability: match_function
756+
required_capability: full_text_functions_in_stats_where
757+
758+
from books
759+
| where length(title) > 40
760+
| stats c = count(*) where match(title, "Lord")
761+
;
762+
763+
c:long
764+
3
765+
;
766+
767+
testMatchInStatsPushableAndNonPushable
768+
required_capability: match_function
769+
required_capability: full_text_functions_in_stats_where
770+
771+
from books
772+
| stats c = count(*) where (match(title, "lord") and ratings > 4.5) or (match(author, "dostoevsky") and length(title) > 50)
773+
;
774+
775+
c:long
776+
6
777+
;
778+
779+
testMatchInStatsPushable
780+
required_capability: match_function
781+
required_capability: full_text_functions_in_stats_where
782+
783+
from books
784+
| stats c = count(*) where match(author, "tolkien")
785+
;
786+
787+
c:long
788+
22
789+
;
790+
791+
testMatchInStatsWithOptions
792+
required_capability: match_function
793+
required_capability: full_text_functions_in_stats_where
794+
795+
FROM books
796+
| STATS c = count(*) where match(title, "Hobbit Back Again", {"operator": "AND"})
797+
;
798+
799+
c:long
800+
1
801+
;
802+
803+
testMatchInStatsWithNonPushableDisjunctions
804+
required_capability: match_function
805+
required_capability: full_text_functions_in_stats_where
806+
807+
FROM books
808+
| STATS c = count(*) where match(title, "lord") or length(title) > 130
809+
;
810+
811+
c:long
812+
5
813+
;
814+
815+
testMatchInStatsWithMultipleAggs
816+
required_capability: match_function
817+
required_capability: full_text_functions_in_stats_where
818+
FROM books
819+
| STATS c = count(*) where match(title, "lord"), m = max(book_no::integer) where match(author, "tolkien"), n = min(book_no::integer) where match(author, "dostoevsky")
820+
;
821+
822+
c:long | m:integer | n:integer
823+
4 | 9607 | 1211
824+
;
825+
826+
827+
testMatchInStatsWithGrouping
828+
required_capability: match_function
829+
required_capability: full_text_functions_in_stats_where
830+
FROM books
831+
| STATS r = AVG(ratings) where match(title, "Lord Rings", {"operator": "AND"}) by author | WHERE r is not null
832+
;
833+
ignoreOrder: true
834+
835+
r:double | author: text
836+
4.75 | Alan Lee
837+
4.674999952316284 | J. R. R. Tolkien
838+
4.670000076293945 | John Ronald Reuel Tolkien
839+
4.670000076293945 | Agnes Perkins
840+
4.670000076293945 | Charles Adolph Huttar
841+
4.670000076293945 | Walter Scheps
842+
4.559999942779541 | J.R.R. Tolkien
843+
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/match-operator.csv-spec

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -751,3 +751,29 @@ from semantic_text
751751
host:keyword | semantic_text_field:text | language_name:keyword | language_code:integer
752752
"host1" | live long and prosper | English | 1
753753
;
754+
755+
756+
testMatchInStatsNonPushable
757+
required_capability: match_operator_colon
758+
required_capability: full_text_functions_in_stats_where
759+
760+
from books
761+
| where length(title) > 40
762+
| stats c = count(*) where title:"Lord"
763+
;
764+
765+
c:long
766+
3
767+
;
768+
769+
testMatchInStatsPushable
770+
required_capability: match_operator_colon
771+
required_capability: full_text_functions_in_stats_where
772+
773+
from books
774+
| stats c = count(*) where author:"tolkien"
775+
;
776+
777+
c:long
778+
22
779+
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/qstr-function.csv-spec

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -210,3 +210,92 @@ book_no:keyword | title:text
210210
7480 | The Hobbit
211211
// end::qstr-with-options-result[]
212212
;
213+
214+
testQstrInStatsNonPushable
215+
required_capability: qstr_function
216+
required_capability: full_text_functions_in_stats_where
217+
218+
from books
219+
| where length(title) > 40
220+
| stats c = count(*) where qstr("title:Lord")
221+
;
222+
223+
c:long
224+
3
225+
;
226+
227+
testMatchInStatsPushableAndNonPushable
228+
required_capability: qstr_function
229+
required_capability: full_text_functions_in_stats_where
230+
231+
from books
232+
| stats c = count(*) where (qstr("title: lord") and ratings > 4.5) or (qstr("author: dostoevsky") and length(title) > 50)
233+
;
234+
235+
c:long
236+
6
237+
;
238+
239+
testQstrInStatsPushable
240+
required_capability: qstr_function
241+
required_capability: full_text_functions_in_stats_where
242+
243+
from books
244+
| stats c = count(*) where qstr("author:tolkien")
245+
;
246+
247+
c:long
248+
22
249+
;
250+
251+
testQstrInStatsWithOptions
252+
required_capability: qstr_function
253+
required_capability: full_text_functions_in_stats_where
254+
255+
FROM books
256+
| STATS c = count(*) where qstr("title: Hobbit Back Again", {"default_operator": "AND"})
257+
;
258+
259+
c:long
260+
1
261+
;
262+
263+
testQstrInStatsWithNonPushableDisjunctions
264+
required_capability: qstr_function
265+
required_capability: full_text_functions_in_stats_where
266+
FROM books
267+
| STATS c = count(*) where qstr("title: lord") or length(title) > 130
268+
;
269+
270+
c:long
271+
5
272+
;
273+
274+
testQstrInStatsWithMultipleAggs
275+
required_capability: qstr_function
276+
required_capability: full_text_functions_in_stats_where
277+
FROM books
278+
| STATS c = count(*) where qstr("title: lord"), m = max(book_no::integer) where qstr("author: tolkien"), n = min(book_no::integer) where qstr("author: dostoevsky")
279+
;
280+
281+
c:long | m:integer | n:integer
282+
4 | 9607 | 1211
283+
;
284+
285+
testQstrInStatsWithGrouping
286+
required_capability: qstr_function
287+
required_capability: full_text_functions_in_stats_where
288+
FROM books
289+
| STATS r = AVG(ratings) where qstr("title: Lord Rings", {"default_operator": "AND"}) by author | WHERE r is not null
290+
;
291+
ignoreOrder: true
292+
293+
r:double | author: text
294+
4.75 | Alan Lee
295+
4.674999952316284 | J. R. R. Tolkien
296+
4.670000076293945 | John Ronald Reuel Tolkien
297+
4.670000076293945 | Agnes Perkins
298+
4.670000076293945 | Charles Adolph Huttar
299+
4.670000076293945 | Walter Scheps
300+
4.559999942779541 | J.R.R. Tolkien
301+
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/scoring.csv-spec

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -521,7 +521,6 @@ book_no:keyword | _score:double
521521
8678 | 0.0
522522
;
523523

524-
525524
disjunctionScoresMultipleClauses
526525

527526
required_capability: metadata_score
@@ -544,3 +543,18 @@ book_no:keyword | _score:double
544543
4023 | 1.5062403678894043
545544
2924 | 1.2732219696044922
546545
;
546+
547+
statsScores
548+
549+
required_capability: metadata_score
550+
required_capability: match_function
551+
required_capability: full_text_functions_in_stats_where
552+
553+
from books metadata _score
554+
| where match(title, "Lord Rings", {"operator": "AND"})
555+
| stats avg_score = avg(_score), max_score = max(_score), min_score = min(_score)
556+
;
557+
558+
avg_score:double | max_score:double | min_score:double
559+
3.869828939437866 | 5.123856544494629 | 3.0124807357788086
560+
;

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/plugin/KqlFunctionIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ public void testKqlQueryWithinEval() {
6666
""";
6767

6868
var error = expectThrows(VerificationException.class, () -> run(query));
69-
assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE commands"));
69+
assertThat(error.getMessage(), containsString("[KQL] function is only supported in WHERE and STATS commands"));
7070
}
7171

7272
public void testInvalidKqlQueryEof() {

0 commit comments

Comments
 (0)