Skip to content

KQL update tests #116483

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 8, 2024
Merged

KQL update tests #116483

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
872314f
Update sample queries to use field names mapped in the tests instead …
afoucret Nov 8, 2024
c90c943
Adding nested fields to the tests mapping.
afoucret Nov 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 54 additions & 1 deletion ...lugin/kql/src/test/java/org/elasticsearch/xpack/kql/parser/AbstractKqlParserTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,12 @@

package org.elasticsearch.xpack.kql.parser;

import org.elasticsearch.common.Strings;
import org.elasticsearch.common.compress.CompressedXContent;
import org.elasticsearch.common.regex.Regex;
import org.elasticsearch.core.Predicates;
import org.elasticsearch.core.SuppressForbidden;
import org.elasticsearch.index.mapper.MapperService;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.query.MultiMatchQueryBuilder;
Expand All @@ -19,6 +22,7 @@
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.index.query.WildcardQueryBuilder;
import org.elasticsearch.test.AbstractBuilderTestCase;
import org.elasticsearch.xcontent.XContentBuilder;

import java.io.BufferedReader;
import java.io.IOException;
Expand All @@ -36,6 +40,7 @@
import java.util.stream.Collectors;
import java.util.stream.Stream;

import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder;
import static org.hamcrest.Matchers.anEmptyMap;
import static org.hamcrest.Matchers.contains;
import static org.hamcrest.Matchers.equalTo;
Expand All @@ -46,6 +51,43 @@ public abstract class AbstractKqlParserTestCase extends AbstractBuilderTestCase
protected static final String UNSUPPORTED_QUERY_FILE_PATH = "/unsupported-queries";
protected static final Predicate<String> BOOLEAN_QUERY_FILTER = (q) -> q.matches("(?i)[^{]*[^\\\\]*(NOT|AND|OR)[^}]*");

protected static final String NESTED_FIELD_NAME = "mapped_nested";

@Override
protected void initializeAdditionalMappings(MapperService mapperService) throws IOException {
XContentBuilder mapping = jsonBuilder().startObject().startObject("_doc").startObject("properties");

mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject();
mapping.startObject(NESTED_FIELD_NAME);
{
mapping.field("type", "nested");
mapping.startObject("properties");
{
mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject();
mapping.startObject(KEYWORD_FIELD_NAME).field("type", "keyword").endObject();
mapping.startObject(INT_FIELD_NAME).field("type", "integer").endObject();
mapping.startObject(NESTED_FIELD_NAME);
{
mapping.field("type", "nested");
mapping.startObject("properties");
{
mapping.startObject(TEXT_FIELD_NAME).field("type", "text").endObject();
Copy link
Contributor

@ioanatia ioanatia Nov 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

quite the nesting 😄

mapping.startObject(KEYWORD_FIELD_NAME).field("type", "keyword").endObject();
mapping.startObject(INT_FIELD_NAME).field("type", "integer").endObject();
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();
}
mapping.endObject();

mapping.endObject().endObject().endObject();

mapperService.merge("_doc", new CompressedXContent(Strings.toString(mapping)), MapperService.MergeReason.MAPPING_UPDATE);
}

protected static String wrapWithRandomWhitespaces(String input) {
return String.join("", randomWhitespaces(), input, randomWhitespaces());
}
Expand Down Expand Up @@ -93,7 +135,18 @@ private static InputStream readFromJarUrl(URL source) throws IOException {
protected List<String> mappedLeafFields() {
return Stream.concat(
Arrays.stream(MAPPED_LEAF_FIELD_NAMES),
List.of(DATE_FIELD_NAME, INT_FIELD_NAME).stream().map(subfieldName -> OBJECT_FIELD_NAME + "." + subfieldName)
Stream.of(
// Adding mapped_object subfields
Strings.format("%s.%s", OBJECT_FIELD_NAME, INT_FIELD_NAME),
Strings.format("%s.%s", OBJECT_FIELD_NAME, DATE_FIELD_NAME),
// Adding mapped_nested subfields
Strings.format("%s.%s", NESTED_FIELD_NAME, TEXT_FIELD_NAME),
Strings.format("%s.%s", NESTED_FIELD_NAME, KEYWORD_FIELD_NAME),
Strings.format("%s.%s", NESTED_FIELD_NAME, INT_FIELD_NAME),
Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, TEXT_FIELD_NAME),
Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, KEYWORD_FIELD_NAME),
Strings.format("%s.%s.%s", NESTED_FIELD_NAME, NESTED_FIELD_NAME, INT_FIELD_NAME)
)
).toList();
}

Expand Down
164 changes: 86 additions & 78 deletions x-pack/plugin/kql/src/test/resources/supported-queries
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,54 +23,54 @@ f*oo
*:"foo bar"

// Querying a field
foo_field:200
foo_field:foo
foo_field:foo bar
foo_field:(foo bar)
foo_field:foo*
foo_field: f*oo
foo_field: *foo
foo_field:"foo bar"
foo_field.subfield:foo
foo_*_field:foo
foo_field:*
foo_*:*
mapped_int:200
Copy link
Contributor

@ioanatia ioanatia Nov 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would have named these int_field, text_field etc - but these were already added in AbstractBuilderTestCase a long time ago, so it's all good.

Copy link
Contributor Author

@afoucret afoucret Nov 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would also prefer it but it is inherited, so I have to deal with these names 😿

mapped_string_2:foo
mapped_string:foo bar
mapped_string:(foo bar)
mapped_string:foo*
mapped_string_2: f*oo
mapped_string: *foo
mapped_string:"foo bar"
mapped_object.subfield:foo
mapped_str*:foo
mapped_string:*
mapped_str_*:*

// Range queries
foo_field<200
foo_field<foo
foo_field<"foo bar"
foo_field>=200
foo_field>=foo
foo_field>"foo bar"
foo_field<=foo
foo_field>=foo
mapped_int<200
mapped_string_2<foo
mapped_string<"foo bar"
mapped_double>=200
mapped_string_alias>=foo
mapped_string>"foo bar"
mapped_string<=foo
mapped_string_2>=foo

// Boolean queries
NOT foo
NOT foo bar
NOT foo_field:foo
NOT foo_field<foo
foo_field:foo AND foo_field:foo bar
foo_field<foo AND foo_field>bar
(foo_field:foo) AND (foo_field:foo bar)
foo_field:foo OR foo_field:foo bar
NOT(foo_field:foo OR foo_field:foo bar)
NOT(foo_field:foo AND foo_field:foo bar)
NOT foo_field:foo AND NOT foo_field:foo bar
(NOT foo_field:foo) AND (NOT foo_field:foo bar)
NOT(foo_field:foo) AND NOT(foo_field:foo bar)
foo_field:foo AND foo_field:foo bar AND foo bar
foo_field:foo AND foo_field:foo bar OR foo bar
foo_field:foo OR foo_field:foo bar OR foo bar
foo_field:foo OR foo_field:foo bar AND foo bar
foo_field:foo AND (foo_field:foo bar OR foo bar)
foo_field:foo AND (foo_field:foo bar OR foo bar)
foo_field:foo OR (foo_field:foo bar OR foo bar)
NOT mapped_string:foo
NOT mapped_string_2<foo
mapped_string:foo AND mapped_string_2:foo bar
mapped_string<foo AND mapped_string>bar
(mapped_string:foo) AND (mapped_string:foo bar)
mapped_string:foo OR mapped_string_2:foo bar
NOT(mapped_string:foo OR mapped_string:foo bar)
NOT(mapped_string:foo AND mapped_string:foo bar)
NOT mapped_string:foo AND NOT mapped_string_2:foo bar
(NOT mapped_string_alias:foo) AND (NOT mapped_string:foo bar)
NOT(mapped_string:foo) AND NOT(mapped_string:foo bar)
mapped_string:foo AND mapped_string_2:foo bar AND foo bar
mapped_string:foo AND mapped_string_2:foo bar OR foo bar
mapped_string:foo OR mapped_string_2:foo bar OR foo bar
mapped_string:foo OR mapped_string:foo bar AND foo bar
mapped_string:foo AND (mapped_string_2:foo bar OR foo bar)
mapped_string:foo AND (mapped_string_2:foo bar OR foo bar)
mapped_string:foo OR (mapped_string_2:foo bar OR foo bar)

foo:AND
foo:OR
foo:NOT
mapped_string:AND
mapped_string:OR
mapped_string:NOT
foo AND
foo OR
foo NOT
Expand All @@ -79,43 +79,51 @@ OR foo
NOT

// Nested queries
nested_field: { NOT foo }
nested_field: { NOT foo bar }
nested_field: { NOT foo_field:foo }
nested_field: { foo_field:foo AND foo_field:foo bar }
nested_field: { foo_field<foo AND foo_field>bar }
nested_field: { (foo_field:foo) AND (foo_field:foo bar) }
nested_field: { foo_field:foo OR foo_field:foo bar }
nested_field: { NOT(foo_field:foo OR foo_field:foo bar) }
nested_field: { NOT(foo_field:foo AND foo_field:foo bar) }
nested_field: { NOT foo_field:foo AND NOT foo_field:foo bar }
nested_field: { (NOT foo_field:foo) AND (NOT foo_field:foo bar) }
nested_field: { NOT(foo_field:foo) AND NOT(foo_field:foo bar) }
nested_field: { foo_field:foo AND foo_field:foo bar AND foo bar }
nested_field: { foo_field:foo AND foo_field:foo bar OR foo bar }
nested_field: { foo_field:foo OR foo_field:foo bar OR foo bar }
nested_field: { foo_field:foo OR foo_field:foo bar AND foo bar }
nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) }
nested_field: { foo_field:foo AND (foo_field:foo bar OR foo bar) }
nested_field: { foo_field:foo OR (foo_field:foo bar OR foo bar) }
nested_field: { sub_nested_field : { foo_field:foo } AND foo_field:foo bar }
mapped_nested: { NOT foo }
mapped_nested: { NOT foo bar }
mapped_nested: { NOT mapped_string:foo }
mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar }
mapped_nested: { mapped_string<foo AND mapped_int>2 }
mapped_nested: { (mapped_string:foo) AND (mapped_string_2:foo bar) }
mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar }
mapped_nested: { NOT(mapped_string:foo OR mapped_string_2:foo bar) }
mapped_nested: { NOT(mapped_string:foo AND mapped_string_2:foo bar) }
mapped_nested: { NOT mapped_string:foo AND NOT mapped_string_2:foo bar }
mapped_nested: { (NOT mapped_string:foo) AND (NOT mapped_string_2:foo bar) }
mapped_nested: { NOT(mapped_string:foo) AND NOT(mapped_string_2:foo bar) }
mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar AND foo bar }
mapped_nested: { mapped_string:foo AND mapped_string_2:foo bar OR foo bar }
mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar OR foo bar }
mapped_nested: { mapped_string:foo OR mapped_string_2:foo bar AND foo bar }
mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) }
mapped_nested: { mapped_string:foo AND (mapped_string_2:foo bar OR foo bar) }
mapped_nested: { mapped_string:foo OR (mapped_string_2:foo bar OR foo bar) }
mapped_nested: { mapped_str*:foo }
mapped_nested: { mapped_nested : { mapped_string:foo AND mapped_int < 3 } AND mapped_string_2:foo bar }
mapped_nested: { mapped_nested.mapped_string:foo AND mapped_string_2:foo bar }

// Inline nested queries
mapped_nested.mapped_string:foo AND mapped_nested.mapped_int < 2
mapped_nested.mapped_nested.mapped_string:foo AND mapped_nested.mapped_int < 2
mapped_nested.mapped_str*: foo


// Queries with escape sequences
foo_field : (foo\(bar\))
foo_field : foo\:bar
foo_field : (foo \\and bar)
foo_field : (foo \\or bar)
foo_field : foo \\not bar
foo_field : foo \{bar\}
foo_field : foo \(bar\)
foo_field : foo \\ bar
foo_field : foo \"bar\"
mapped_string:(foo\(bar\))
mapped_string:foo\:bar
mapped_string:(foo \\and bar)
mapped_string:(foo \\or bar)
mapped_string:foo \\not bar
mapped_string:foo \{bar\}
mapped_string:foo \(bar\)
mapped_string:foo \\ bar
mapped_string:foo \"bar\"

foo_field : "foo and bar"
foo_field : "foo not bar"
foo_field : "foo or bar"
foo_field : "foo : bar"
foo_field : "foo { bar }"
foo_field : "foo (bar)"
foo_field : "foo \\ bar"
foo_field : "foo \"bar\""
mapped_string:"foo and bar"
mapped_string:"foo not bar"
mapped_string:"foo or bar"
mapped_string:"foo : bar"
mapped_string:"foo { bar }"
mapped_string:"foo (bar)"
mapped_string:"foo \\ bar"
mapped_string:"foo \"bar\""
42 changes: 21 additions & 21 deletions x-pack/plugin/kql/src/test/resources/unsupported-queries
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,36 +1,36 @@

// Incomplete expressions
foo_field :
foo_field <
foo_field >
foo_field >=
foo_field <=
mapped_string :
mapped_string <
mapped_string >
mapped_string >=
mapped_string <=
>= foo
: "foo"
: foo

// Parentheses mismatch
foo_field: (foo bar
foo_field: foo bar)
NOT foo_field:foo OR foo_field:foo bar)
NOT (foo_field:foo AND) foo_field:foo bar
mapped_string: (foo bar
mapped_string: foo bar)
NOT mapped_string:foo OR mapped_string_2:foo bar)
NOT (mapped_string:foo AND) mapped_string_2:foo bar

// Quotes mismatch
foo_field: "foo bar
foo_field: foo bar"
mapped_string: "foo bar
mapped_string: foo bar"

// Can't nest grouping terms parentheses
foo_field:(foo (bar))
mapped_string:(foo (bar))

// Bad syntax for nested fields:
nested_field { foo: bar }
mapped_nested { mapped_string: bar }

// Missing escape sequences:
foo_field: foo:bar
foo_field: (foo and bar)
foo_field: (foo or bar)
foo_field: foo not bar
foo_field: foo { bar }
foo_field: foo (bar)
foo_field: foo "bar"
foo_field: "foo "bar""
mapped_string: foo:bar
mapped_string: (foo and bar)
mapped_string: (foo or bar)
mapped_string: foo not bar
mapped_string: foo { bar }
mapped_string: foo (bar)
mapped_string: foo "bar"
mapped_string: "foo "bar""