diff --git a/x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java b/x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java index 718410667c5e3..b062ee0eb16da 100644 --- a/x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java +++ b/x-pack/plugin/eql/qa/mixed-node/src/test/java/org/elasticsearch/xpack/eql/qa/mixed_node/EqlSearchIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.eql.qa.mixed_node; import org.apache.http.HttpHost; +import org.elasticsearch.Version; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; @@ -17,8 +18,11 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.test.NotEqualMessageBuilder; import org.elasticsearch.test.rest.ESRestTestCase; +import org.elasticsearch.xpack.eql.execution.search.RuntimeUtils; +import org.elasticsearch.xpack.eql.expression.function.EqlFunctionRegistry; import org.elasticsearch.xpack.ql.TestNode; import org.elasticsearch.xpack.ql.TestNodes; +import org.elasticsearch.xpack.ql.expression.function.FunctionDefinition; import org.junit.After; import org.junit.Before; @@ -26,8 +30,11 @@ import java.io.InputStream; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; import static java.util.Arrays.asList; import static java.util.Collections.emptyMap; @@ -94,6 +101,91 @@ public void testSequencesWithRequestToUpgradedNodes() throws Exception { assertSequncesQueryOnNodes(newNodes); } + /** + * Requests are sent to the new (upgraded) version nodes of the cluster. The request should be redirected to the old nodes at this point + * if their version is lower than {@code org.elasticsearch.xpack.eql.execution.search.RuntimeUtils.SWITCH_TO_MULTI_VALUE_FIELDS_VERSION} + * version. + */ + public void testMultiValueFields() throws Exception { + final String bulkEntries = readResource(EqlSearchIT.class.getResourceAsStream("/eql_data.json")); + Request bulkRequst = new Request("POST", index + "/_bulk?refresh"); + bulkRequst.setJsonEntity(bulkEntries); + assertOK(client().performRequest(bulkRequst)); + + // build a set of functions names to check if all functions are tested with multi-value fields + final Set availableFunctions = new EqlFunctionRegistry().listFunctions() + .stream() + .map(FunctionDefinition::name) + .collect(Collectors.toSet()); + // each function has a query and query results associated to it + Set testedFunctions = new HashSet<>(); + // TODO: remove the 8.0.0 version check after code reaches 7.x as well + boolean multiValued = newNodes.get(0).getVersion() != Version.V_8_0_0 + && nodes.getBWCVersion().onOrAfter(RuntimeUtils.SWITCH_TO_MULTI_VALUE_FIELDS_VERSION); + try ( + // TODO: use newNodes (instead of bwcNodes) after code reaches 7.x as well + RestClient client = buildClient(restClientSettings(), + bwcNodes.stream().map(TestNode::getPublishAddress).toArray(HttpHost[]::new)) + ) { + // filter only the relevant bits of the response + String filterPath = "filter_path=hits.events._id"; + Request request = new Request("POST", index + "/_eql/search?" + filterPath); + + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "between", + "PROCESS where between(process_name, \\\"w\\\", \\\"s\\\") : \\\"indow\\\"", + multiValued ? new int[] {120, 121} : new int[] {121}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "cidrmatch", + "PROCESS where string(cidrmatch(source_address, \\\"10.6.48.157/24\\\")) : \\\"true\\\"", + multiValued ? new int[] {121, 122} : new int[] {122}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "concat", + "PROCESS where concat(file_name, process_name) == \\\"foo\\\" or add(pid, ppid) > 100", + multiValued ? new int[] {116, 117, 120, 121, 122} : new int[] {120, 121}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "endswith", + "PROCESS where string(endswith(process_name, \\\"s\\\")) : \\\"true\\\"", + multiValued ? new int[] {120, 121} : new int[] {121}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "indexof", + "PROCESS where indexof(file_name, \\\"x\\\", 2) > 0", + multiValued ? new int[] {116, 117} : new int[] {117}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "length", + "PROCESS where length(file_name) >= 3 and length(file_name) == 1", + multiValued ? new int[] {116} : new int[] {}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "startswith", + "PROCESS where string(startswith~(file_name, \\\"F\\\")) : \\\"true\\\"", + multiValued ? new int[] {116, 117, 120, 121} : new int[] {116, 120, 121}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "string", + "PROCESS where string(concat(file_name, process_name) == \\\"foo\\\") : \\\"true\\\"", + multiValued ? new int[] {116, 120} : new int[] {120}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "stringcontains", + "PROCESS where string(stringcontains(file_name, \\\"txt\\\")) : \\\"true\\\"", + multiValued ? new int[] {117} : new int[] {}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "substring", + "PROCESS where substring(file_name, -4) : \\\".txt\\\"", + multiValued ? new int[] {117} : new int[] {}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "add", + "PROCESS where add(pid, 1) == 2", + multiValued ? new int[] {120, 121, 122} : new int[] {120, 121, 122}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "divide", + "PROCESS where divide(pid, 12) == 1", + multiValued ? new int[] {116, 117, 118, 119, 120, 122} : new int[] {116, 117, 118, 119}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "modulo", + "PROCESS where modulo(ppid, 10) == 0", + multiValued ? new int[] {121, 122} : new int[] {121}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "multiply", + "PROCESS where multiply(pid, 10) == 120", + multiValued ? new int[] {116, 117, 118, 119, 120, 122} : new int[] {116, 117, 118, 119, 120, 122}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "number", + "PROCESS where number(command_line) + pid >= 360", + multiValued ? new int[] {122, 123} : new int[] {123}); + assertMultiValueFunctionQuery(availableFunctions, testedFunctions, request, client, "subtract", + "PROCESS where subtract(pid, 1) == 0", + multiValued ? new int[] {120, 121, 122} : new int[] {120, 121, 122}); + } + + // check that ALL functions from the function registry have a test query. We don't want to miss any of the functions, since this + // is about painless scripting + assertTrue(testedFunctions.containsAll(availableFunctions)); + } + private void assertEventsQueryOnNodes(List nodesList) throws Exception { final String event = randomEvent(); Map expectedResponse = prepareEventsTestData(event); @@ -222,6 +314,17 @@ private Map prepareSequencesTestData() throws IOException { return expectedResponse; } + private void assertMultiValueFunctionQuery(Set availableFunctions, Set testedFunctions, Request request, + RestClient client, String functionName, String query, int[] ids) throws IOException { + List eventIds = new ArrayList<>(); + for (int id : ids) { + eventIds.add(String.valueOf(id)); + } + request.setJsonEntity("{\"query\":\"" + query + "\"}"); + assertResponse(query, eventIds, runEql(client, request)); + testedFunctions.add(functionName); + } + private void assertResponse(Map expected, Map actual) { if (false == expected.equals(actual)) { NotEqualMessageBuilder message = new NotEqualMessageBuilder(); @@ -230,6 +333,29 @@ private void assertResponse(Map expected, Map ac } } + @SuppressWarnings("unchecked") + private void assertResponse(String query, List expected, Map actual) { + List> events = new ArrayList<>(); + Map hits = (Map) actual.get("hits"); + if (hits == null || hits.isEmpty()) { + if (expected.isEmpty()) { + return; + } + fail("For query [" + query + "]\nResponse does not match: the returned list of resuts is empty.\nExpected " + expected); + } else { + events = (List>) hits.get("events"); + } + + List actualList = new ArrayList<>(); + events.stream().forEach(m -> actualList.add(m.get("_id"))); + + if (false == expected.equals(actualList)) { + NotEqualMessageBuilder message = new NotEqualMessageBuilder(); + message.compareLists(actualList, expected); + fail("For query [" + query + "]\nResponse does not match:\n" + message.toString()); + } + } + private Map runEql(RestClient client, Request request) throws IOException { Response response = client.performRequest(request); try (InputStream content = response.getEntity().getContent()) { diff --git a/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json index 3fc6e28834ea8..23ef72f382272 100644 --- a/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json +++ b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_data.json @@ -28,3 +28,19 @@ {"@timestamp":"12345678914","event_type":"success","sequence":44,"correlation_success1":"C","correlation_success2":"D"} {"index":{"_id":15}} {"@timestamp":"12345678999","event_type":"failure","sequence":44,"correlation_failure1":"C","correlation_failure2":"D"} +{"index":{"_id":116}} +{"@timestamp":"123456789116","event_type":"PROCESS","file_name":["x","f","zyx"],"process_name":["oo","abc"],"pid":[12,13,14],"ppid":1} +{"index":{"_id":117}} +{"@timestamp":"123456789117","event_type":"PROCESS","file_name":["a.exe","f.txt"],"process_name":"oo","pid":[12,13,14],"ppid":[89,1,2,3]} +{"index":{"_id":118}} +{"@timestamp":"123456789118","event_type":"PROCESS","file_name":"a","process_name":"oo","pid":12} +{"index":{"_id":119}} +{"@timestamp":"123456789119","event_type":"PROCESS","process_name":["oo","foo"],"pid":[121,12]} +{"index":{"_id":120}} +{"@timestamp":"123456789120","event_type":"PROCESS","file_name":["f","g","f"],"process_name":["oo","pp","windows"],"pid":[12,1,2,333],"ppid":121} +{"index":{"_id":121}} +{"@timestamp":"123456789121","event_type":"PROCESS","file_name":"f","pid":1,"ppid":[100,1000],"source_address":["127.0.0.1","10.6.48.157","10.0.0.5"],"process_name":"windows"} +{"index":{"_id":122}} +{"@timestamp":"123456789122","event_type":"PROCESS","pid":[1,2,3,4,5,6,12],"ppid":[66,67,68,69,99,100],"source_address":"10.6.48.157","command_line":"348"} +{"index":{"_id":123}} +{"@timestamp":"123456789123","event_type":"PROCESS","pid":500,"command_line":"100"} diff --git a/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_mapping.json b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_mapping.json index f56dea6722183..d8428f229b1c3 100644 --- a/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_mapping.json +++ b/x-pack/plugin/eql/qa/mixed-node/src/test/resources/eql_mapping.json @@ -31,5 +31,23 @@ "path": "sequence" } } + }, + "command_line": { + "type": "keyword" + }, + "file_name": { + "type": "keyword" + }, + "process_name": { + "type": "keyword" + }, + "pid" : { + "type" : "long" + }, + "ppid" : { + "type" : "long" + }, + "source_address" : { + "type" : "ip" } } diff --git a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java index 94ce58dce9cd1..46d2a9740a82b 100644 --- a/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java +++ b/x-pack/plugin/eql/src/internalClusterTest/java/org/elasticsearch/xpack/eql/action/AsyncEqlSearchActionIT.java @@ -332,8 +332,8 @@ public static class FakePainlessScriptPlugin extends MockScriptPlugin { @Override protected Map, Object>> pluginScripts() { Map, Object>> scripts = new HashMap<>(); - scripts.put("InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.div(" + - "params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", FakePainlessScriptPlugin::fail); + scripts.put("InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0 -> InternalQlScriptUtils.nullSafeFilter(" + + "InternalQlScriptUtils.eq(InternalQlScriptUtils.div(params.v1,X0),params.v2)))", FakePainlessScriptPlugin::fail); return scripts; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java index f4ce9cf954201..9baa70c95b51a 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/RuntimeUtils.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; @@ -41,11 +42,11 @@ import java.util.Set; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.xpack.ql.execution.search.QlSourceBuilder.SWITCH_TO_FIELDS_API_VERSION; public final class RuntimeUtils { static final Logger QUERY_LOG = LogManager.getLogger(QueryClient.class); + public static final Version SWITCH_TO_MULTI_VALUE_FIELDS_VERSION = Version.V_7_15_0; private RuntimeUtils() {} @@ -148,7 +149,7 @@ public static HitExtractor createExtractor(FieldExtraction ref, EqlConfiguration public static SearchRequest prepareRequest(SearchSourceBuilder source, boolean includeFrozen, String... indices) { - SearchRequest searchRequest = new SearchRequest(SWITCH_TO_FIELDS_API_VERSION); + SearchRequest searchRequest = new SearchRequest(SWITCH_TO_MULTI_VALUE_FIELDS_VERSION); searchRequest.indices(indices); searchRequest.source(source); searchRequest.allowPartialSearchResults(false); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java index 85d25e735da79..d76be851a1c88 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/expression/function/scalar/whitelist/InternalEqlScriptUtils.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.eql.expression.function.scalar.whitelist; +import org.elasticsearch.index.fielddata.ScriptDocValues; import org.elasticsearch.xpack.eql.expression.function.scalar.math.ToNumberFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.BetweenFunctionProcessor; import org.elasticsearch.xpack.eql.expression.function.scalar.string.CIDRMatchFunctionProcessor; @@ -20,6 +21,8 @@ import org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQlScriptUtils; import java.util.List; +import java.util.Map; +import java.util.function.Predicate; import static org.elasticsearch.xpack.eql.expression.predicate.operator.comparison.InsensitiveBinaryComparisonProcessor.InsensitiveBinaryComparisonOperation; @@ -33,6 +36,20 @@ public class InternalEqlScriptUtils extends InternalQlScriptUtils { InternalEqlScriptUtils() { } + public static Boolean multiValueDocValues(Map> doc, String fieldName, Predicate script) { + ScriptDocValues docValues = doc.get(fieldName); + if (docValues != null && docValues.isEmpty() == false) { + for (T value : docValues) { + if (script.test(value)) { + return true; + } + } + return false; + } + // missing value means "null" + return script.test(null); + } + public static Boolean seq(Object left, Object right) { return InsensitiveBinaryComparisonOperation.SEQ.apply(left, right); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java index 787e745c0354d..061bb31552cb3 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/optimizer/Optimizer.java @@ -29,7 +29,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.Predicates; import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; -import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; @@ -40,12 +39,10 @@ import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BinaryComparisonSimplification; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanFunctionEqualsElimination; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.BooleanSimplification; -import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineBinaryComparisons; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.CombineDisjunctionsToIn; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ConstantFolding; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.LiteralsOnTheRight; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.OptimizerRule; -import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateEquals; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneLiteralsInOrderBy; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PushDownAndCombineFilters; import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.ReplaceSurrogateFunction; @@ -66,7 +63,6 @@ import static java.util.Arrays.asList; import static java.util.stream.Collectors.toList; -import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PropagateNullable; public class Optimizer extends RuleExecutor { @@ -85,14 +81,10 @@ protected Iterable.Batch> batches() { Batch operators = new Batch("Operator Optimization", new ConstantFolding(), // boolean - new BooleanSimplification(), + new EqlBooleanSimplification(), new LiteralsOnTheRight(), new BinaryComparisonSimplification(), new BooleanFunctionEqualsElimination(), - // needs to occur before BinaryComparison combinations - new PropagateEquals(), - new PropagateNullable(), - new CombineBinaryComparisons(), new CombineDisjunctionsToIn(), new SimplifyComparisonsArithmetics(DataTypes::areCompatible), // prune/elimination @@ -179,10 +171,9 @@ protected LogicalPlan rule(Filter filter) { comparableToNull = cmp.right(); } if (comparableToNull != null) { - if (cmp instanceof Equals) { - result = new IsNull(cmp.source(), comparableToNull); - } else { - result = new IsNotNull(cmp.source(), comparableToNull); + result = new IsNull(cmp.source(), comparableToNull); + if (cmp instanceof Equals == false) { + result = new Not(cmp.source(), result); } } } @@ -200,6 +191,19 @@ protected Expression regexToEquals(RegexMatch regexMatch, Literal literal) { } } + private static class EqlBooleanSimplification extends BooleanSimplification { + + EqlBooleanSimplification() { + super(); + } + + @Override + protected Expression maybeSimplifyNegatable(Expression e) { + return null; + } + + } + static class PruneFilters extends org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PruneFilters { @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java index 191f0c5de15fe..860a56894cf87 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/ExpressionBuilder.java @@ -46,7 +46,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataType; @@ -146,7 +145,7 @@ public Expression visitComparison(ComparisonContext ctx) { case EqlBaseParser.EQ: return new Equals(source, left, right, zoneId); case EqlBaseParser.NEQ: - return new NotEquals(source, left, right, zoneId); + return new Not(source, new Equals(source, left, right, zoneId)); case EqlBaseParser.LT: return new LessThan(source, left, right, zoneId); case EqlBaseParser.LTE: diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/MultiValueAwareScriptQuery.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/MultiValueAwareScriptQuery.java new file mode 100644 index 0000000000000..d8b18b6ed14cd --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/MultiValueAwareScriptQuery.java @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.eql.planner; + +import org.elasticsearch.xpack.ql.expression.gen.script.ScriptTemplate; +import org.elasticsearch.xpack.ql.querydsl.query.ScriptQuery; +import org.elasticsearch.xpack.ql.tree.Source; + +class MultiValueAwareScriptQuery extends ScriptQuery { + + MultiValueAwareScriptQuery(Source source, ScriptTemplate script) { + super(source, script); + } + + @Override + protected ScriptTemplate nullSafeScript(ScriptTemplate script) { + return script; + } + +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java index b5b422a71e684..afd1de6da05d6 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/planner/QueryTranslator.java @@ -17,9 +17,12 @@ import org.elasticsearch.xpack.ql.expression.Expression; import org.elasticsearch.xpack.ql.expression.Expressions; import org.elasticsearch.xpack.ql.expression.FieldAttribute; +import org.elasticsearch.xpack.ql.expression.Literal; +import org.elasticsearch.xpack.ql.expression.function.Function; import org.elasticsearch.xpack.ql.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.ql.expression.function.scalar.string.BinaryComparisonCaseInsensitiveFunction; import org.elasticsearch.xpack.ql.expression.function.scalar.string.CaseInsensitiveScalarFunction; +import org.elasticsearch.xpack.ql.expression.gen.script.Scripts; import org.elasticsearch.xpack.ql.expression.predicate.logical.And; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; import org.elasticsearch.xpack.ql.planner.ExpressionTranslator; @@ -68,9 +71,23 @@ public static Query toQuery(Expression e, TranslatorHandler handler) { for (ExpressionTranslator translator : QUERY_TRANSLATORS) { translation = translator.translate(e, handler); if (translation != null) { - return translation; + break; } } + if (translation != null) { + if (translation instanceof ScriptQuery) { + // check the operators and the expressions involved in these operations so that all can be used + // in a doc-values multi-valued context + boolean multiValuedIncompatible = e.anyMatch(exp -> { + return false == (exp instanceof Literal || exp instanceof FieldAttribute || exp instanceof Function); + }); + if (multiValuedIncompatible == false) { + ScriptQuery query = (ScriptQuery) translation; + return new MultiValueAwareScriptQuery(query.source(), Scripts.multiValueDocValuesRewrite(query.script())); + } + } + return translation; + } throw new QlIllegalArgumentException("Don't know how to translate {} {}", e.nodeName(), e); } diff --git a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt index d0300aeac7696..c3722d377f39d 100644 --- a/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt +++ b/x-pack/plugin/eql/src/main/resources/org/elasticsearch/xpack/eql/plugin/eql_whitelist.txt @@ -65,6 +65,7 @@ class org.elasticsearch.xpack.ql.expression.function.scalar.whitelist.InternalQl class org.elasticsearch.xpack.eql.expression.function.scalar.whitelist.InternalEqlScriptUtils { + Boolean multiValueDocValues(java.util.Map, String, java.util.function.Predicate) # # ASCII Functions # diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java index 793007458fed1..279c85f61ac52 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/optimizer/OptimizerTests.java @@ -33,8 +33,8 @@ import org.elasticsearch.xpack.ql.expression.Order.NullsPosition; import org.elasticsearch.xpack.ql.expression.Order.OrderDirection; import org.elasticsearch.xpack.ql.expression.predicate.logical.And; +import org.elasticsearch.xpack.ql.expression.predicate.logical.Not; import org.elasticsearch.xpack.ql.expression.predicate.logical.Or; -import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNotNull; import org.elasticsearch.xpack.ql.expression.predicate.nulls.IsNull; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.GreaterThan; @@ -43,6 +43,7 @@ import org.elasticsearch.xpack.ql.expression.predicate.regex.Like; import org.elasticsearch.xpack.ql.index.EsIndex; import org.elasticsearch.xpack.ql.index.IndexResolution; +import org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PushDownAndCombineFilters; import org.elasticsearch.xpack.ql.plan.TableIdentifier; import org.elasticsearch.xpack.ql.plan.logical.Filter; import org.elasticsearch.xpack.ql.plan.logical.LogicalPlan; @@ -67,7 +68,6 @@ import static org.elasticsearch.xpack.eql.EqlTestUtils.TEST_CFG; import static org.elasticsearch.xpack.ql.TestUtils.UTC; import static org.elasticsearch.xpack.ql.expression.Literal.TRUE; -import static org.elasticsearch.xpack.ql.optimizer.OptimizerRules.PushDownAndCombineFilters; import static org.elasticsearch.xpack.ql.tree.Source.EMPTY; import static org.elasticsearch.xpack.ql.type.DataTypes.INTEGER; @@ -129,9 +129,13 @@ public void testIsNotNull() { Filter filter = (Filter) plan; And condition = (And) filter.condition(); - assertTrue(condition.right() instanceof IsNotNull); + assertTrue(condition.right() instanceof Not); + Not not = (Not) condition.right(); + List children = not.children(); + assertEquals(1, children.size()); + assertTrue(children.get(0) instanceof IsNull); - IsNotNull check = (IsNotNull) condition.right(); + IsNull check = (IsNull) children.get(0); assertEquals(((FieldAttribute) check.field()).name(), "command_line"); } } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java index 842bba1d729a0..ab9526f8e206d 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.In; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThan; import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.LessThanOrEqual; -import org.elasticsearch.xpack.ql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.ql.tree.Source; import org.elasticsearch.xpack.ql.type.DataTypes; @@ -403,7 +402,7 @@ public void testComparison() { Expression value = expr(valueText); assertEquals(new Equals(null, field, value, UTC), expr(fieldText + "==" + valueText)); - assertEquals(new NotEquals(null, field, value, UTC), expr(fieldText + "!=" + valueText)); + assertEquals(new Not(null, new Equals(null, field, value, UTC)), expr(fieldText + "!=" + valueText)); assertEquals(new LessThanOrEqual(null, field, value, UTC), expr(fieldText + "<=" + valueText)); assertEquals(new GreaterThanOrEqual(null, field, value, UTC), expr(fieldText + ">=" + valueText)); assertEquals(new GreaterThan(null, field, value, UTC), expr(fieldText + ">" + valueText)); diff --git a/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt b/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt index 3364cc4db97f7..f3f158b605e4f 100644 --- a/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt +++ b/x-pack/plugin/eql/src/test/resources/querytranslator_tests.txt @@ -93,13 +93,28 @@ mixedTypeFilter process where process_name : "notepad.exe" or (serial_event_id < 4.5 and serial_event_id >= 3.1) ; "term":{"process_name":{"value":"notepad.exe","case_insensitive":true,"boost":1.0} -"range":{"serial_event_id":{"from":3.1,"to":4.5,"include_lower":true,"include_upper":false +{"bool":{"must":[{"range":{"serial_event_id":{"from":null,"to":4.5,"include_lower":false,"include_upper":false +{"range":{"serial_event_id":{"from":3.1,"to":null,"include_lower":true,"include_upper":false ; notFilter process where not (exit_code > -1) ; -"range":{"exit_code":{"from":null,"to":-1,"include_lower":false,"include_upper":true +{"bool":{"must_not":[{"range":{"exit_code":{"from":-1,"to":null,"include_lower":false,"include_upper":false +; + +notFieldsConjunction +process where not (exit_code > 1 and user_name == "root") +; +{"bool":{"must_not":[{"bool":{"must":[{"range":{"exit_code":{"from":1,"to":null,"include_lower":false,"include_upper":false +{"term":{"user_name":{"value":"root" +; + +notOneFieldConjunctionNotSecondField +process where not (exit_code > 1) and not (user_name == "root") +; +{"bool":{"must":[{"bool":{"must_not":[{"range":{"exit_code":{"from":1,"to":null,"include_lower":false,"include_upper":false +{"bool":{"must_not":[{"term":{"user_name":{"value":"root" ; inFilter @@ -108,6 +123,20 @@ process where process_name in ("python.exe", "SMSS.exe", "explorer.exe") "terms":{"process_name":["python.exe","SMSS.exe","explorer.exe"], ; +inFilterWithScripting +process where substring(command_line, 5) in ("test*","best") +; +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.in(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3)))" +"params":{"v0":"command_line","v1":5,"v2":null,"v3":["test*","best"]}} +; + +negatedInFilterWithScripting +process where substring(command_line, 5) not in ("test*","best") +; +{"script":{"script":{"source":"InternalQlScriptUtils.not(InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.in(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3))))" +"params":{"v0":"command_line","v1":5,"v2":null,"v3":["test*","best"]}} +; + equalsAndInFilter process where process_path : "*\\red_ttp\\wininit.*" and opcode in (0,1,2,3) ; @@ -137,6 +166,15 @@ process where process_name in~ ("test*", "best") "term":{"process_name":{"value":"best","case_insensitive":true,"boost":1.0} ; +inFilterInsensitiveWithScripting +process where substring(command_line, 5) in~ ("test*", "best") +; +{"bool":{"should":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3)))" +"params":{"v0":"command_line","v1":5,"v2":null,"v3":"test*"}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3)))" +"params":{"v0":"command_line","v1":5,"v2":null,"v3":"best"}} +; + functionEqualsTrue process where cidrMatch(source_address, "10.0.0.0/8") == true @@ -247,24 +285,24 @@ process where bool != false lengthFunctionWithExactSubField process where length(file_name) > 0 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( -InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalEqlScriptUtils.length(X0),params.v1)))", "params":{"v0":"file_name.keyword","v1":0} ; lengthFunctionWithExactField process where 12 == length(user_name) ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.length(X0),params.v1)))", "params":{"v0":"user_name","v1":12} ; lengthFunctionWithConstantKeyword process where 5 > length(constant_keyword) ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lt( -InternalEqlScriptUtils.length(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lt(InternalEqlScriptUtils.length(X0),params.v1)))", "params":{"v0":"constant_keyword","v1":5} ; @@ -337,57 +375,54 @@ process where stringContains(process_name, "foo") stringFunction process where string(pid) : "123" ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq( -InternalEqlScriptUtils.string(InternalQlScriptUtils.docValue(doc,params.v0)),params.v1))", -"params":{"v0":"pid","v1":"123"} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.string(X0),params.v1)))" +"params":{"v0":"pid","v1":"123"}} ; indexOfFunction-caseSensitive process where indexOf(user_name, "A", 2) > 0 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( -InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3),params.v4))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( +InternalEqlScriptUtils.indexOf(X0,params.v1,params.v2,params.v3),params.v4)))", "params":{"v0":"user_name","v1":"A","v2":2,"v3":false,"v4":0} ; indexOfFunction-insensitive process where indexOf~(user_name, "A", 2) > 0 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( -InternalEqlScriptUtils.indexOf(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3),params.v4))", +script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt( +InternalEqlScriptUtils.indexOf(X0,params.v1,params.v2,params.v3),params.v4)))", "params":{"v0":"user_name","v1":"A","v2":2,"v3":true,"v4":0} ; substringFunction process where substring(file_name, -4) : ".exe" ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq( -InternalEqlScriptUtils.substring(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", -"params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"} +"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3)))" +"params":{"v0":"file_name.keyword","v1":-4,"v2":null,"v3":".exe"}} ; betweenFunction process where between(process_name, "s", "e") : "yst" ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq( -InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3,params.v4),params.v5))", -"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.between(X0,params.v1,params.v2,params.v3,params.v4),params.v5)))" +"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":false,"v5":"yst"}} ; betweenFunction-insensitive process where between~(process_name, "s", "e") : "yst" ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq( -InternalEqlScriptUtils.between(InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3,params.v4),params.v5))", -"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":true,"v5":"yst"} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.between(X0,params.v1,params.v2,params.v3,params.v4),params.v5)))" +"params":{"v0":"process_name","v1":"s","v2":"e","v3":false,"v4":true,"v5":"yst"}} ; concatFunction process where concat(process_name, "::foo::", null, 1) : "net.exe::foo::1" ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq( -InternalEqlScriptUtils.concat([InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2,params.v3]),params.v4))", -"params":{"v0":"process_name","v1":"::foo::","v2":null,"v3":1,"v4":"net.exe::foo::1"} +"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.concat([X0,params.v1,params.v2,params.v3]),params.v4)))" +"params":{"v0":"process_name","v1":"::foo::","v2":null,"v3":1,"v4":"net.exe::foo::1"}} ; cidrMatchFunctionOne @@ -428,37 +463,40 @@ process where cidrMatch(source_address, "10.0.0.0/8", "192.168.0.0/16", "2001:db cidrMatchFunctionWrapped process where string(cidrMatch(source_address, "10.6.48.157/8")) : "true" ; -{"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.string( -InternalEqlScriptUtils.cidrMatch(InternalQlScriptUtils.docValue(doc,params.v0),params.v1)),params.v2))" -"params":{"v0":"source_address","v1":["10.6.48.157/8"],"v2":"true"} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalEqlScriptUtils.seq(InternalEqlScriptUtils.string(InternalEqlScriptUtils.cidrMatch(X0,params.v1)),params.v2)))" +"params":{"v0":"source_address","v1":["10.6.48.157/8"],"v2":"true"}} ; numberFunctionSingleArgument -process where number(process_name) == 1; -InternalEqlScriptUtils.number(InternalQlScriptUtils.docValue(doc,params.v0),params.v1) -"params":{"v0":"process_name","v1":null,"v2":1} +process where number(process_name) == 1 +; +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.number(X0,params.v1),params.v2)))" +"params":{"v0":"process_name","v1":null,"v2":1}} ; - numberFunctionTwoFieldArguments -process where number(process_name, pid) != null; -InternalEqlScriptUtils.number(InternalQlScriptUtils.docValue(doc,params.v0),InternalQlScriptUtils.docValue(doc,params.v1))))", -"params":{"v0":"process_name","v1":"pid"} +process where number(process_name, pid) != null +; +{"script":{"script":{"source":"InternalQlScriptUtils.not(InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalEqlScriptUtils.multiValueDocValues(doc,params.v1,X1->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.isNull(InternalEqlScriptUtils.number(X0,X1))))))" +"params":{"v0":"process_name","v1":"pid"}} ; numberFunctionTwoArguments -process where number(process_name, 16) != null; -InternalEqlScriptUtils.number(InternalQlScriptUtils.docValue(doc,params.v0),params.v1) -"params":{"v0":"process_name","v1":16} +process where number(process_name, 16) != null +; +{"script":{"script":{"source":"InternalQlScriptUtils.not(InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.isNull(InternalEqlScriptUtils.number(X0,params.v1)))))" +"params":{"v0":"process_name","v1":16}} ; numberFunctionFoldedComparison -process where serial_event_id == number("-32.5"); +process where serial_event_id == number("-32.5") +; {"term":{"serial_event_id":{"value":-32.5,"boost":1.0} ; numberFunctionFoldedHexComparison -process where serial_event_id == number("0x32", 16); +process where serial_event_id == number("0x32", 16) +; {"term":{"serial_event_id":{"value":50,"boost":1.0} ; @@ -466,161 +504,161 @@ process where serial_event_id == number("0x32", 16); addOperator process where serial_event_id + 2 == -2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.add(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":-2147483647} ; addOperatorReversed process where 2 + serial_event_id == -2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.add(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":-2147483647} ; addFunction process where add(serial_event_id, 2) == -2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.add(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":-2147483647} ; addFunctionReversed process where add(2, serial_event_id) == -2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.add(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.add(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":-2147483647} ; divideOperator process where serial_event_id / 2 == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.div(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.div(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; divideOperatorReversed process where 82 / serial_event_id == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.div(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", -"params":{"v0":82,"v1":"serial_event_id","v2":41} +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.div(params.v1,X0),params.v2)))", +"params":{"v0":"serial_event_id","v1":82,"v2":41} ; divideFunction process where divide(serial_event_id, 2) == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.div(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.div(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; divideFunctionReversed process where divide(82, serial_event_id) == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.div(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", -"params":{"v0":82,"v1":"serial_event_id","v2":41} +{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.div(params.v1,X0),params.v2)))", +"params":{"v0":"serial_event_id","v1":82,"v2":41} ; moduloOperator process where serial_event_id % 2 == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mod(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mod(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; moduloOperatorReversed process where 42 % serial_event_id == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mod(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", -"params":{"v0":42,"v1":"serial_event_id","v2":41} +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mod(params.v1,X0),params.v2)))", +"params":{"v0":"serial_event_id","v1":42,"v2":41} ; moduloFunction process where modulo(serial_event_id, 2) == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mod(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mod(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; moduloFunctionReversed process where modulo(42, serial_event_id) == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mod(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", -"params":{"v0":42,"v1":"serial_event_id","v2":41} +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mod(params.v1,X0),params.v2)))", +"params":{"v0":"serial_event_id","v1":42,"v2":41} ; multiplyOperator process where serial_event_id * 2 == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mul(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; multiplyOperatorReversed process where 2 * serial_event_id == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mul(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; multiplyFunction process where multiply(serial_event_id, 2) == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mul(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; multiplyFunctionReversed process where multiply(2, serial_event_id) == 41 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.mul(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.mul(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":41} ; subtractOperator process where serial_event_id - 2 == 2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.sub(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.sub(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":2147483647} ; subtractOperatorReversed process where 43 - serial_event_id == -2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.sub(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", -"params":{"v0":43,"v1":"serial_event_id","v2":-2147483647} +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.sub(params.v1,X0),params.v2)))", +"params":{"v0":"serial_event_id","v1":43,"v2":-2147483647} ; subtractFunction process where subtract(serial_event_id, 2) == 2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.sub(InternalQlScriptUtils.docValue(doc,params.v0),params.v1),params.v2))", +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.sub(X0,params.v1),params.v2)))", "params":{"v0":"serial_event_id","v1":2,"v2":2147483647} ; subtractFunctionReversed process where subtract(43, serial_event_id) == -2147483647 ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq( -InternalQlScriptUtils.sub(params.v0,InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))", -"params":{"v0":43,"v1":"serial_event_id","v2":-2147483647} +"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0, +X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalQlScriptUtils.sub(params.v1,X0),params.v2)))", +"params":{"v0":"serial_event_id","v1":43,"v2":-2147483647} ; eventQueryDefaultLimit @@ -695,18 +733,18 @@ process where command_line like~ ("n?t.e?e", "net.*") likeMultiArgWithScript process where substring(command_line, 5) like ("net.e*", "net.e?e") ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", +{"bool":{"should":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3)))" "params":{"v0":"command_line","v1":5,"v2":null,"v3":"^net\\.e.*$"}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3)))" "params":{"v0":"command_line","v1":5,"v2":null,"v3":"^net\\.e.e$"}} ; likeMultiArgWithScriptInsensitive process where substring(command_line, 5) like~ ("net.e*", "net.e?e") ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3,params.v4))", +{"bool":{"should":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3,params.v4)))" "params":{"v0":"command_line","v1":5,"v2":null,"v3":"^net\\.e.*$","v4":true}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3,params.v4)))" "params":{"v0":"command_line","v1":5,"v2":null,"v3":"^net\\.e.e$","v4":true}} ; @@ -761,16 +799,105 @@ process where command_line regex~ ("^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\ regexMultiArgWithScript process where substring(command_line, 5) regex ("^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+") ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3))", +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3)))" "params":{"v0":"command_line","v1":5,"v2":null,"v3":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+"}} ; regexMultiArgWithScriptInsensitive process where substring(command_line, 5) regex~ ("^.*?net.exe", "net\\.exe", "C:\\\\Windows\\\\system32\\\\net1\\s+") ; -"script":{"source":"InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring( -InternalQlScriptUtils.docValue(doc,params.v0),params.v1,params.v2),params.v3,params.v4))", +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.regex(InternalEqlScriptUtils.substring(X0,params.v1,params.v2),params.v3,params.v4)))" "params":{"v0":"command_line","v1":5,"v2":null,"v3":"^.*?net.exe|net\\.exe|C:\\\\Windows\\\\system32\\\\net1\\s+","v4":true}} ; +conjunctionOfLengthFunctions +process where length(file_name) > 0 and length(process_name) <= 0 +; +{"bool":{"must":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"file_name.keyword","v1":0}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lte(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"process_name","v1":0}} +; + +disjunctionOfLengthFunctions +process where length(file_name) > 0 or length(process_name) <= 0 +; +{"bool":{"should":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"file_name.keyword","v1":0}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lte(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"process_name","v1":0}} +; + +conjunctionOfMultiFieldsFunctions +process where concat(file_name, ".", process_name) == "foo" and add(pid, ppid) > 100 +; +{"bool":{"must":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalEqlScriptUtils.multiValueDocValues(doc,params.v1,X1->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.concat([X0,params.v2,X1]),params.v3))))" +"params":{"v0":"file_name","v1":"process_name","v2":".","v3":"foo"}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalEqlScriptUtils.multiValueDocValues(doc,params.v1,X1->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalQlScriptUtils.add(X0,X1),params.v2))))" +"params":{"v0":"pid","v1":"ppid","v2":100}} +; + +disjunctionOfMultiFieldsFunctions +process where concat(file_name, ".", process_name) == "foo" or add(pid, ppid) > 100 +; +{"bool":{"should":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalEqlScriptUtils.multiValueDocValues(doc,params.v1,X1->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.concat([X0,params.v2,X1]),params.v3))))" +"params":{"v0":"file_name","v1":"process_name","v2":".","v3":"foo"}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalEqlScriptUtils.multiValueDocValues(doc,params.v1,X1->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalQlScriptUtils.add(X0,X1),params.v2))))" +"params":{"v0":"pid","v1":"ppid","v2":100}} +; + +// the conjunction will be applied on different values of the multi-value field +conjunctionOfSameFieldSameFunction +process where length(file_name) > 5 and length(file_name) < 10 +; +{"bool":{"must":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"file_name.keyword","v1":5}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lt(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"file_name.keyword","v1":10}} +; + +// the disjunction will be applied on different values of the multi-value field +disjunctionOfSameFieldSameFunction +process where length(file_name) > 5 or length(file_name) < 10 +; +{"bool":{"should":[{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"file_name.keyword","v1":5}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.lt(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"file_name.keyword","v1":10}} +; + +isNullInScript +process where null == (exit_code > -1) +; +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.isNull(InternalQlScriptUtils.gt(X0,params.v1))))" +"params":{"v0":"exit_code","v1":-1}} +; + +isNull +process where pid != null +; +{"bool":{"must_not":[{"bool":{"must_not":[{"exists":{"field":"pid" +; + +disjunctionOfFunctionAndNegatedFunction +process where not (length(user) == 1) and length(user) == 1 +; +{"bool":{"must":[{"script":{"script":{"source":"InternalQlScriptUtils.not(InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.length(X0),params.v1))))" +"params":{"v0":"user","v1":1}} +{"script":{"script":{"source":"InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X0->InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.eq(InternalEqlScriptUtils.length(X0),params.v1)))" +"params":{"v0":"user","v1":1}} +; + +twoRangesOnASingleField +// in a multi-value fields scenario where we want different values of the same field to obey the two conditions we need to use +// two range queries, because a single range query will apply the two conditions on the same value of the multi-value field. +process where pid > 100 and pid < 200 +; +{"bool":{"must":[{"range":{"pid":{"from":100,"to":null,"include_lower":false,"include_upper":false +{"range":{"pid":{"from":null,"to":200,"include_lower":false,"include_upper":false +; + +// this type of query (disjunction/conjunction inside a function where one of the conditions is negated) is currently unsupported because +// the "not" has an undefined location of where it should be used in the Painless script +// disjunctionInsideFunctionWithNot +// process where string(pid > 5 and pid != 10) == \"true\" \ No newline at end of file diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java index d56513ff4ac92..a80386491140c 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/expression/gen/script/Scripts.java @@ -12,10 +12,13 @@ import org.elasticsearch.xpack.ql.util.Check; import java.util.AbstractMap.SimpleEntry; +import java.util.ArrayList; import java.util.LinkedHashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Stream; @@ -31,6 +34,8 @@ public final class Scripts { public static final String EQL_SCRIPTS = "{eql}"; public static final String SQL_SCRIPTS = "{sql}"; public static final String PARAM = "{}"; + static final String DOC_VALUE_PARAMS_REGEX = + "InternalQlScriptUtils\\.docValue\\(doc,(params\\.%s)\\)|(params\\.%s)|InternalQlScriptUtils\\.not\\("; public static final String INTERNAL_QL_SCRIPT_UTILS = "InternalQlScriptUtils"; public static final String INTERNAL_EQL_SCRIPT_UTILS = "InternalEqlScriptUtils"; public static final String INTERNAL_SQL_SCRIPT_UTILS = "InternalSqlScriptUtils"; @@ -47,6 +52,7 @@ private Scripts() { new SimpleEntry<>(SQL_SCRIPTS, INTERNAL_SQL_SCRIPT_UTILS), new SimpleEntry<>(PARAM, "params.%s")) .collect(toMap(e -> Pattern.compile(e.getKey(), Pattern.LITERAL), Map.Entry::getValue, (a, b) -> a, LinkedHashMap::new))); + static final Pattern qlDocValuePattern = Pattern.compile(DOC_VALUE_PARAMS_REGEX); /** * Expands common tokens inside the script: @@ -107,4 +113,117 @@ public static String classPackageAsPrefix(Class function) { Check.isTrue(index > 0, "invalid package {}", prefix); return "{" + prefix.substring(0, index) + "}"; } + + /** + * This method replaces any .docValue(doc,params.%s) call with a "Xn" variable. + * Each variable is then used in a {@code java.util.function.Predicate} to iterate over the doc_values in a Painless script. + * Multiple .docValue(doc,params.%s) calls for the same field will use multiple .docValue calls, meaning + * a different value of the field will be used for each usage in the script. + * + * For example, a query of the form fieldA - fieldB > 0 that gets translated into the following Painless script + * {@code InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalQlScriptUtils.sub( + * InternalQlScriptUtils.docValue(doc,params.v0),InternalQlScriptUtils.docValue(doc,params.v1)),params.v2))} + * will become, after this method rewrite + * {@code InternalEqlScriptUtils.multiValueDocValues(doc,params.v0,X1 -> InternalEqlScriptUtils.multiValueDocValues(doc,params.v1, + * X2 -> InternalQlScriptUtils.nullSafeFilter(InternalQlScriptUtils.gt(InternalQlScriptUtils.sub(X1,X2),params.v2))))} + */ + public static ScriptTemplate multiValueDocValuesRewrite(ScriptTemplate script) { + return docValuesRewrite(script, false); + } + + private static ScriptTemplate docValuesRewrite(ScriptTemplate script, boolean useSameValueInScript) { + int index = 0; // counts how many params there are to be able to get their value from script's params + Map params = script.params().asParams(); + List fieldVars = new ArrayList<>(); + List otherVars = new ArrayList<>(); + StringBuilder newTemplate = new StringBuilder(); + int negated = 0; + + String[] tokens = splitWithMatches(script.template(), qlDocValuePattern); + for (String token : tokens) { + // A docValue call will be replaced with "X" followed by a counter + // The scripts that come into this method can contain multiple docValue calls for the same field + // This method will use only one variable for one docValue call + if ("InternalQlScriptUtils.docValue(doc,params.%s)".equals(token)) { + Object fieldName = params.get("v" + index); + + if (useSameValueInScript) { + // if the field is already in our list, don't add it one more time + if (fieldVars.contains(fieldName) == false) { + fieldVars.add(fieldName); + } + newTemplate.append("X" + fieldVars.indexOf(fieldName)); + } else { + fieldVars.add(fieldName); + newTemplate.append("X" + (fieldVars.size() - 1)); + } + // increase the params position + index++; + } else if ("InternalQlScriptUtils.not(".equals(token)) { + negated++; + } else if ("params.%s".equals(token)) { + newTemplate.append(token); + // gather the other type of params (which are not docValues calls) so that at the end we rebuild the list of params + otherVars.add(params.get("v" + index)); + index++; + } else { + newTemplate.append(token); + } + for (int i = 0; i < negated - 1; i++) { + // remove this many closing parantheses as "InternalQlScriptUtils.not(" matches found, minus one + newTemplate.deleteCharAt(newTemplate.length() - 1); + } + } + + // iterate over the fields in reverse order and add a multiValueDocValues call for each + for(int i = fieldVars.size() - 1; i >= 0; i--) { + newTemplate.insert(0, "InternalEqlScriptUtils.multiValueDocValues(doc,params.%s,X" + i + " -> "); + newTemplate.append(")"); + } + if (negated > 0) { + newTemplate.insert(0, "InternalQlScriptUtils.not("); + } + + ParamsBuilder newParams = paramsBuilder(); + // field variables are first + fieldVars.forEach(v -> newParams.variable(v)); + // the rest of variables come after + otherVars.forEach(v -> newParams.variable(v)); + + return new ScriptTemplate(newTemplate.toString(), newParams.build(), DataTypes.BOOLEAN); + } + + /* + * Split a string given a regular expression into tokens. The list of tokens includes both the + * segments that matched the regex and also the segments that didn't. + * "fooxbarxbaz" split using the "x" regex will build an array like ["foo","x","bar","x","baz"] + */ + static String[] splitWithMatches(String input, Pattern pattern) { + int index = 0; + ArrayList matchList = new ArrayList<>(); + Matcher m = pattern.matcher(input); + + while(m.find()) { + if (index != m.start()) { + matchList.add(input.subSequence(index, m.start()).toString()); // add the segment before the match + } + if (m.start() != m.end()) { + matchList.add(input.subSequence(m.start(), m.end()).toString()); // add the match itself + } + index = m.end(); + } + + // if no match was found, return this + if (index == 0) { + return new String[] {input}; + } + + // add remaining segment and avoid an empty element in matches list + if (index < input.length()) { + matchList.add(input.subSequence(index, input.length()).toString()); + } + + // construct result + return matchList.toArray(new String[matchList.size()]); + } } diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java index 00762ad497ee4..acc9d74e2d3d5 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/optimizer/OptimizerRules.java @@ -126,7 +126,7 @@ protected Expression rule(BinaryComparison bc) { } } - public static final class BooleanSimplification extends OptimizerExpressionRule { + public static class BooleanSimplification extends OptimizerExpressionRule { public BooleanSimplification() { super(TransformDirection.UP); @@ -238,8 +238,9 @@ private Expression simplifyNot(Not n) { return new Literal(n.source(), Boolean.TRUE, DataTypes.BOOLEAN); } - if (c instanceof Negatable) { - return ((Negatable) c).negate(); + Expression negated = maybeSimplifyNegatable(c); + if (negated != null) { + return negated; } if (c instanceof Not) { @@ -248,6 +249,17 @@ private Expression simplifyNot(Not n) { return n; } + + /** + * @param e + * @return the negated expression or {@code null} if the parameter is not an instance of {@code Negatable} + */ + protected Expression maybeSimplifyNegatable(Expression e) { + if (e instanceof Negatable) { + return ((Negatable) e).negate(); + } + return null; + } } public static class BinaryComparisonSimplification extends OptimizerExpressionRule { diff --git a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/ScriptQuery.java b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/ScriptQuery.java index 2de01c3e3d076..61935247a6b8b 100644 --- a/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/ScriptQuery.java +++ b/x-pack/plugin/ql/src/main/java/org/elasticsearch/xpack/ql/querydsl/query/ScriptQuery.java @@ -22,7 +22,7 @@ public class ScriptQuery extends LeafQuery { public ScriptQuery(Source source, ScriptTemplate script) { super(source); // make script null safe - this.script = Scripts.nullSafeFilter(script); + this.script = nullSafeScript(script); } public ScriptTemplate script() { @@ -57,4 +57,8 @@ public boolean equals(Object obj) { protected String innerToString() { return script.toString(); } + + protected ScriptTemplate nullSafeScript(ScriptTemplate script) { + return Scripts.nullSafeFilter(script); + } } diff --git a/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/script/ScriptsTests.java b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/script/ScriptsTests.java new file mode 100644 index 0000000000000..836dbf9d02dc8 --- /dev/null +++ b/x-pack/plugin/ql/src/test/java/org/elasticsearch/xpack/ql/expression/gen/script/ScriptsTests.java @@ -0,0 +1,70 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.ql.expression.gen.script; + +import org.elasticsearch.test.ESTestCase; + +import java.util.regex.Pattern; + +public class ScriptsTests extends ESTestCase { + + public void testSplitWithMatches() { + String input = "A1B2C3"; + Pattern pattern = Pattern.compile("[0-9]+"); + + assertArrayEquals(new String[] {"A", "1", "B", "2", "C", "3"}, Scripts.splitWithMatches(input, pattern)); + } + + public void testSplitWithMatchesNoMatches() { + String input = "AxBxCx"; + Pattern pattern = Pattern.compile("[0-9]+"); + + assertArrayEquals(new String[] {input}, Scripts.splitWithMatches(input, pattern)); + } + + public void testSplitWithMatchesOneMatch() { + String input = "ABC"; + Pattern pattern = Pattern.compile("ABC"); + + assertArrayEquals(new String[] {input}, Scripts.splitWithMatches(input, pattern)); + } + + public void testSplitWithMatchesSameMatch() { + String input = "xxxx"; + Pattern pattern = Pattern.compile("x"); + + assertArrayEquals(new String[] {"x","x","x","x"}, Scripts.splitWithMatches(input, pattern)); + } + + public void testSplitWithMatchesTwoPatterns() { + String input = "xyxy"; + Pattern pattern = Pattern.compile("x|y"); + + assertArrayEquals(new String[] {"x","y","x","y"}, Scripts.splitWithMatches(input, pattern)); + } + + public void testSplitWithMatchesTwoPatterns2() { + String input = "A1B2C3"; + Pattern pattern = Pattern.compile("[0-9]{1}|[A-F]{1}"); + + assertArrayEquals(new String[] {"A", "1", "B", "2", "C", "3"}, Scripts.splitWithMatches(input, pattern)); + } + + public void testSplitWithMatchesTwoPatterns3() { + String input = "A111BBB2C3"; + Pattern pattern = Pattern.compile("[0-9]+|[A-F]+"); + + assertArrayEquals(new String[] {"A", "111", "BBB", "2", "C", "3"}, Scripts.splitWithMatches(input, pattern)); + } + + public void testSplitWithMatchesTwoPatterns4() { + String input = "xA111BxBB2C3x"; + Pattern pattern = Pattern.compile("[0-9]+|[A-F]+"); + + assertArrayEquals(new String[] {"x", "A", "111", "B", "x", "BB", "2", "C", "3", "x"}, Scripts.splitWithMatches(input, pattern)); + } +}