EQL: Return sequence join keys in the original type (#61268) (#61283)

(cherry picked from commit d54957d)

client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchResponse.java

@@ -24,7 +24,9 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.InstantiatingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 
@@ -146,20 +148,21 @@ private static final class Fields {
             new ConstructingObjectParser<>("eql/search_response_sequence", true,
                 args -> {
                     int i = 0;
-                    @SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
+                    @SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
                     @SuppressWarnings("unchecked") List<SearchHit> events = (List<SearchHit>) args[i];
                     return new EqlSearchResponse.Sequence(joinKeys, events);
                 });
 
         static {
-            PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), JOIN_KEYS);
+            PARSER.declareFieldArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p),
+                JOIN_KEYS, ObjectParser.ValueType.VALUE_ARRAY);
             PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> SearchHit.fromXContent(p), EVENTS);
         }
 
-        private final List<String> joinKeys;
+        private final List<Object> joinKeys;
         private final List<SearchHit> events;
 
-        public Sequence(List<String> joinKeys, List<SearchHit> events) {
+        public Sequence(List<Object> joinKeys, List<SearchHit> events) {
             this.joinKeys = joinKeys == null ? Collections.emptyList() : joinKeys;
             this.events = events == null ? Collections.emptyList() : events;
         }
@@ -186,7 +189,7 @@ public int hashCode() {
             return Objects.hash(joinKeys, events);
         }
 
-        public List<String> joinKeys() {
+        public List<Object> joinKeys() {
             return joinKeys;
         }
 
@@ -204,7 +207,7 @@ private static final class Fields {
         }
 
         private final int count;
-        private final List<String> keys;
+        private final List<Object> keys;
         private final float percent;
 
         private static final ParseField COUNT = new ParseField(Fields.COUNT);
@@ -216,18 +219,19 @@ private static final class Fields {
                 args -> {
                     int i = 0;
                     int count = (int) args[i++];
-                    @SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
+                    @SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
                     float percent = (float) args[i];
                     return new EqlSearchResponse.Count(count, joinKeys, percent);
                 });
 
         static {
             PARSER.declareInt(ConstructingObjectParser.constructorArg(), COUNT);
-            PARSER.declareStringArray(ConstructingObjectParser.constructorArg(), KEYS);
+            PARSER.declareFieldArray(constructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p), KEYS,
+                ObjectParser.ValueType.VALUE_ARRAY);
             PARSER.declareFloat(ConstructingObjectParser.constructorArg(), PERCENT);
         }
 
-        public Count(int count, List<String> keys, float percent) {
+        public Count(int count, List<Object> keys, float percent) {
             this.count = count;
             this.keys = keys == null ? Collections.emptyList() : keys;
             this.percent = percent;
@@ -260,7 +264,7 @@ public int count() {
             return count;
         }
 
-        public List<String> keys() {
+        public List<Object> keys() {
             return keys;
         }
 

client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java

@@ -30,6 +30,7 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
+import java.util.function.Supplier;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -69,11 +70,12 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomS
         int size = randomIntBetween(1, 10);
         List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence> seq = null;
         if (randomBoolean()) {
+            List<Supplier<Object[]>> randoms = getKeysGenerators();
             seq = new ArrayList<>();
             for (int i = 0; i < size; i++) {
-                List<String> joins = null;
+                List<Object> joins = null;
                 if (randomBoolean()) {
-                    joins = Arrays.asList(generateRandomStringArray(6, 11, false));
+                    joins = Arrays.asList(randomFrom(randoms).get());
                 }
                 seq.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(joins, randomEvents()));
             }
@@ -90,15 +92,26 @@ public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomS
         }
     }
 
+    private static List<Supplier<Object[]>> getKeysGenerators() {
+        List<Supplier<Object[]>> randoms = new ArrayList<>();
+        randoms.add(() -> generateRandomStringArray(6, 11, false));
+        randoms.add(() -> randomArray(0, 6, Integer[]::new, ()-> randomInt()));
+        randoms.add(() -> randomArray(0, 6, Long[]::new, ()-> randomLong()));
+        randoms.add(() -> randomArray(0, 6, Boolean[]::new, ()-> randomBoolean()));
+
+        return randoms;
+    }
+
     public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomCountResponse(TotalHits totalHits) {
         int size = randomIntBetween(1, 10);
         List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Count> cn = null;
         if (randomBoolean()) {
+            List<Supplier<Object[]>> randoms = getKeysGenerators();
             cn = new ArrayList<>();
             for (int i = 0; i < size; i++) {
-                List<String> keys = null;
+                List<Object> keys = null;
                 if (randomBoolean()) {
-                    keys = Arrays.asList(generateRandomStringArray(6, 11, false));
+                    keys = Arrays.asList(randomFrom(randoms).get());
                 }
                 cn.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Count(randomIntBetween(0, 41), keys, randomFloat()));
             }

docs/reference/eql/detect-threats-with-eql.asciidoc

@@ -328,7 +328,7 @@ The query matches a sequence, indicating the attack likely succeeded.
     "sequences": [
       {
         "join_keys": [
-          "2012"
+          2012
         ],
         "events": [
           {

docs/reference/eql/eql-search-api.asciidoc

@@ -405,7 +405,7 @@ a <<eql-sequences,sequence>>.
 [%collapsible%open]
 =====
 `join_keys`::
-(array of strings)
+(array of values)
 Shared field values used to constrain matches in the sequence. These are defined
 using the <<eql-sequences,`by` keyword>> in the EQL query syntax.
 
@@ -629,7 +629,7 @@ shared `process.pid` value for each matching event.
     "sequences": [
       {
         "join_keys": [
-          "2012"
+          2012
         ],
         "events": [
           {

docs/reference/eql/eql.asciidoc

@@ -320,7 +320,7 @@ contains the shared `process.pid` value for each matching event.
     "sequences": [
       {
         "join_keys": [
-          "2012"
+          2012
         ],
         "events": [
           {

x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml

@@ -11,6 +11,26 @@ setup:
               - category: process
             "@timestamp": 2020-02-03T12:34:56Z
             user: SYSTEM
+            id: 123
+            valid: false
+          - index:
+              _index: eql_test
+              _id:    2
+          - event:
+              - category: process
+            "@timestamp": 2020-02-04T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
+          - index:
+              _index: eql_test
+              _id:    3
+          - event:
+              - category: process
+            "@timestamp": 2020-02-05T12:34:56Z
+            user: SYSTEM
+            id: 123
+            valid: true
 
 ---
 # Testing round-trip and the basic shape of the response
@@ -22,9 +42,60 @@ setup:
           query: "process where user = 'SYSTEM'"
 
   - match: {timed_out: false}
-  - match: {hits.total.value: 1}
+  - match: {hits.total.value: 3}
   - match: {hits.total.relation: "eq"}
   - match: {hits.events.0._source.user: "SYSTEM"}
+  - match: {hits.events.0._id: "1"}
+  - match: {hits.events.1._id: "2"}
+  - match: {hits.events.2._id: "3"}
+
+---
+"Execute EQL sequence with string key.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: "sequence by user [process where user = 'SYSTEM'] [process where true]"
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.1.join_keys.0: "SYSTEM"}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.1._id: "3"}
+
+---
+"Execute EQL sequence with numeric key.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: "sequence by id [process where user = 'SYSTEM'] [process where true]"
+  - match: {timed_out: false}
+  - match: {hits.total.value: 2}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: 123}
+  - match: {hits.sequences.0.events.0._id: "1"}
+  - match: {hits.sequences.0.events.1._id: "2"}
+  - match: {hits.sequences.1.join_keys.0: 123}
+  - match: {hits.sequences.1.events.0._id: "2"}
+  - match: {hits.sequences.1.events.1._id: "3"}
+
+---
+"Execute EQL sequence with boolean key.":
+  - do:
+      eql.search:
+        index: eql_test
+        body:
+          query: "sequence by valid [process where user = 'SYSTEM'] [process where true]"
+  - match: {timed_out: false}
+  - match: {hits.total.value: 1}
+  - match: {hits.total.relation: "eq"}
+  - match: {hits.sequences.0.join_keys.0: true}
+  - match: {hits.sequences.0.events.0._id: "2"}
+  - match: {hits.sequences.0.events.1._id: "3"}
 
 ---
 "Execute some EQL in async mode":
@@ -47,7 +118,7 @@ setup:
   - match: {is_running: false}
   - match: {is_partial: false}
   - match: {timed_out: false}
-  - match: {hits.total.value: 1}
+  - match: {hits.total.value: 3}
   - match: {hits.total.relation: "eq"}
   - match: {hits.events.0._source.user: "SYSTEM"}
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java

@@ -16,10 +16,12 @@
 import org.elasticsearch.common.lucene.Lucene;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.InstantiatingObjectParser;
+import org.elasticsearch.common.xcontent.ObjectParser;
 import org.elasticsearch.common.xcontent.ToXContentFragment;
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.SearchHits;
 
@@ -192,26 +194,28 @@ private static final class Fields {
             new ConstructingObjectParser<>("eql/search_response_sequence", true,
                 args -> {
                     int i = 0;
-                    @SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
+                    @SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
                     @SuppressWarnings("unchecked") List<SearchHit> events = (List<SearchHit>) args[i];
                     return new EqlSearchResponse.Sequence(joinKeys, events);
                 });
 
         static {
-            PARSER.declareStringArray(ConstructingObjectParser.optionalConstructorArg(), JOIN_KEYS);
+            PARSER.declareFieldArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p),
+                JOIN_KEYS, ObjectParser.ValueType.VALUE_ARRAY);
             PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> SearchHit.fromXContent(p), EVENTS);
         }
 
-        private final List<String> joinKeys;
+        private final List<Object> joinKeys;
         private final List<SearchHit> events;
 
-        public Sequence(List<String> joinKeys, List<SearchHit> events) {
+        public Sequence(List<Object> joinKeys, List<SearchHit> events) {
             this.joinKeys = joinKeys == null ? Collections.emptyList() : joinKeys;
             this.events = events == null ? Collections.emptyList() : events;
         }
 
+        @SuppressWarnings("unchecked")
         public Sequence(StreamInput in) throws IOException {
-            this.joinKeys = in.readStringList();
+            this.joinKeys = (List<Object>) in.readGenericValue();
             this.events = in.readList(SearchHit::new);
         }
 
@@ -221,7 +225,7 @@ public static Sequence fromXContent(XContentParser parser) {
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
-            out.writeStringCollection(joinKeys);
+            out.writeGenericValue(joinKeys);
             out.writeList(events);
         }
 
@@ -260,7 +264,7 @@ public int hashCode() {
             return Objects.hash(joinKeys, events);
         }
 
-        public List<String> joinKeys() {
+        public List<Object> joinKeys() {
             return joinKeys;
         }
 
@@ -278,7 +282,7 @@ private static final class Fields {
         }
 
         private final int count;
-        private final List<String> keys;
+        private final List<Object> keys;
         private final float percent;
 
         private static final ParseField COUNT = new ParseField(Fields.COUNT);
@@ -290,26 +294,28 @@ private static final class Fields {
                 args -> {
                     int i = 0;
                     int count = (int) args[i++];
-                    @SuppressWarnings("unchecked") List<String> joinKeys = (List<String>) args[i++];
+                    @SuppressWarnings("unchecked") List<Object> joinKeys = (List<Object>) args[i++];
                     float percent = (float) args[i];
                     return new EqlSearchResponse.Count(count, joinKeys, percent);
                 });
 
         static {
             PARSER.declareInt(constructorArg(), COUNT);
-            PARSER.declareStringArray(constructorArg(), KEYS);
+            PARSER.declareFieldArray(constructorArg(), (p, c) -> XContentParserUtils.parseFieldsValue(p), KEYS,
+                ObjectParser.ValueType.VALUE_ARRAY);
             PARSER.declareFloat(constructorArg(), PERCENT);
         }
 
-        public Count(int count, List<String> keys, float percent) {
+        public Count(int count, List<Object> keys, float percent) {
             this.count = count;
             this.keys = keys == null ? Collections.emptyList() : keys;
             this.percent = percent;
         }
 
+        @SuppressWarnings("unchecked")
         public Count(StreamInput in) throws IOException {
             count = in.readVInt();
-            keys = in.readStringList();
+            keys = (List<Object>) in.readGenericValue();
             percent = in.readFloat();
         }
 
@@ -320,7 +326,7 @@ public static Count fromXContent(XContentParser parser) {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             out.writeVInt(count);
-            out.writeStringCollection(keys);
+            out.writeGenericValue(keys);
             out.writeFloat(percent);
         }
 
@@ -357,7 +363,7 @@ public int count() {
             return count;
         }
 
-        public List<String> keys() {
+        public List<Object> keys() {
             return keys;
         }