EQL: Add optional fields and limit joining keys on non-null values on…

…ly (#79677)

Add optional fields, with usage inside queries' filters and as join
keys. Optional fields have a question mark in front of their name
(`?some_field`) and can be used in standalone event queries and
sequences. If the field exists in at least one index that's getting
queried, that field will actually be used in queries. If the field
doesn't exist in any of the indices, all its mentions in query filters
will be replaced with `null`. For sequences, optional fields as keys can
have `null` as a value, whereas non-optional fields will be restricted
to non-null values only. For example, a query like

```
sequence by ?process.entity_id, process.pid
  [process where transID == 2]
  [file where transID == 0] with runs=2
```

can return a sequence with a join key `[null,123]`. If the sequence will
use `process.pid` as an optional field (`sequence by ?process.entity_id,
?process.pid`), as well, the sequence can now return join keys as
`[null,123]`, `[null,null]`, `[512,null]`.

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/BaseEqlSpecTestCase.java

@@ -45,6 +45,11 @@ public abstract class BaseEqlSpecTestCase extends RemoteClusterAwareEqlRestTestC
     private final String query;
     private final String name;
     private final long[] eventIds;
+    /**
+     * Join keys can be of multiple types, but toml is very restrictive and doesn't allow mixed types values in the same array of values
+     * For now, every value will be converted to a String.
+     */
+    private final String[] joinKeys;
 
     @Before
     public void setup() throws Exception {
@@ -79,18 +84,19 @@ protected static List<Object[]> asArray(List<EqlSpec> specs) {
                 name = "" + (counter);
             }
 
-            results.add(new Object[] { spec.query(), name, spec.expectedEventIds() });
+            results.add(new Object[] { spec.query(), name, spec.expectedEventIds(), spec.joinKeys() });
         }
 
         return results;
     }
 
-    BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds) {
+    BaseEqlSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
         this.index = index;
 
         this.query = query;
         this.name = name;
         this.eventIds = eventIds;
+        this.joinKeys = joinKeys;
     }
 
     public void test() throws Exception {
@@ -192,6 +198,44 @@ protected void assertSequences(List<Sequence> sequences) {
                 .flatMap(s -> s.events().stream())
                 .collect(toList());
         assertEvents(events);
+        List<Object> keys = sequences.stream()
+                .flatMap(s -> s.joinKeys().stream())
+                .collect(toList());
+        assertEvents(events);
+        assertJoinKeys(keys);
+    }
+
+    private void assertJoinKeys(List<Object> keys) {
+        logger.debug("Join keys {}", new Object() {
+            public String toString() {
+                return keysToString(keys);
+            }
+        });
+
+        if (joinKeys == null || joinKeys.length == 0) {
+            return;
+        }
+        String[] actual = new String[keys.size()];
+        int i = 0;
+        for (Object key : keys) {
+            if (key == null) {
+                actual[i] = "null";
+            } else {
+                actual[i] = key.toString();
+            }
+            i++;
+        }
+        assertArrayEquals(LoggerMessageFormat.format(null, "unexpected result for spec[{}] [{}] -> {} vs {}", name, query,
+            Arrays.toString(joinKeys), Arrays.toString(actual)), joinKeys, actual);
+    }
+
+    private String keysToString(List<Object> keys) {
+        StringJoiner sj = new StringJoiner(",", "[", "]");
+        for (Object key : keys) {
+            sj.add(key.toString());
+            sj.add("\n");
+        }
+        return sj.toString();
     }
 
     @Override

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlDateNanosSpecTestCase.java

@@ -21,13 +21,13 @@ public static List<Object[]> readTestSpecs() throws Exception {
     }
 
     // constructor for "local" rest tests
-    public EqlDateNanosSpecTestCase(String query, String name, long[] eventIds) {
-        this(TEST_NANOS_INDEX, query, name, eventIds);
+    public EqlDateNanosSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
+        this(TEST_NANOS_INDEX, query, name, eventIds, joinKeys);
     }
 
     // constructor for multi-cluster tests
-    public EqlDateNanosSpecTestCase(String index, String query, String name, long[] eventIds) {
-        super(index, query, name, eventIds);
+    public EqlDateNanosSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
+        super(index, query, name, eventIds, joinKeys);
     }
 
     @Override

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlExtraSpecTestCase.java

@@ -21,13 +21,13 @@ public static List<Object[]> readTestSpecs() throws Exception {
     }
 
     // constructor for "local" rest tests
-    public EqlExtraSpecTestCase(String query, String name, long[] eventIds) {
-        this(TEST_EXTRA_INDEX, query, name, eventIds);
+    public EqlExtraSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
+        this(TEST_EXTRA_INDEX, query, name, eventIds, joinKeys);
     }
 
     // constructor for multi-cluster tests
-    public EqlExtraSpecTestCase(String index, String query, String name, long[] eventIds) {
-        super(index, query, name, eventIds);
+    public EqlExtraSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
+        super(index, query, name, eventIds, joinKeys);
     }
 
     @Override

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpec.java

@@ -19,6 +19,7 @@ public class EqlSpec {
     private String[] tags;
     private String query;
     private long[] expectedEventIds;
+    private String[] joinKeys;
 
     public String name() {
         return name;
@@ -68,6 +69,14 @@ public void expectedEventIds(long[] expectedEventIds) {
         this.expectedEventIds = expectedEventIds;
     }
 
+    public String[] joinKeys() {
+        return joinKeys;
+    }
+
+    public void joinKeys(String[] joinKeys) {
+        this.joinKeys = joinKeys;
+    }
+
     @Override
     public String toString() {
         String str = "";
@@ -83,6 +92,10 @@ public String toString() {
         if (expectedEventIds != null) {
             str = appendWithComma(str, "expected_event_ids", Arrays.toString(expectedEventIds));
         }
+
+        if (joinKeys != null) {
+            str = appendWithComma(str, "join_keys", Arrays.toString(joinKeys));
+        }
         return str;
     }
 

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecLoader.java

@@ -101,6 +101,9 @@ private static List<EqlSpec> readFromStream(InputStream is, Set<String> uniqueTe
                 }
                 spec.expectedEventIds(expectedEventIds);
             }
+
+            arr = table.getList("join_keys");
+            spec.joinKeys(arr != null ? arr.toArray(new String[0]) : new String[0]);
             validateAndAddSpec(testSpecs, spec, uniqueTestNames);
         }
 

x-pack/plugin/eql/qa/common/src/main/java/org/elasticsearch/test/eql/EqlSpecTestCase.java

@@ -28,12 +28,12 @@ protected String tiebreaker() {
     }
 
     // constructor for "local" rest tests
-    public EqlSpecTestCase(String query, String name, long[] eventIds) {
-        this(TEST_INDEX, query, name, eventIds);
+    public EqlSpecTestCase(String query, String name, long[] eventIds, String[] joinKeys) {
+        this(TEST_INDEX, query, name, eventIds, joinKeys);
     }
 
     // constructor for multi-cluster tests
-    public EqlSpecTestCase(String index, String query, String name, long[] eventIds) {
-        super(index, query, name, eventIds);
+    public EqlSpecTestCase(String index, String query, String name, long[] eventIds, String[] joinKeys) {
+        super(index, query, name, eventIds, joinKeys);
     }
 }

x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml

@@ -430,3 +430,63 @@ query = '''
 process where substring(command_line, 5) regex (".*?net[1]?  localgroup.*?", ".*? myappserver.py .*?")
 '''
 
+[[queries]]
+description = "This query has an important meaning when considering its pair - sequenceWithRequiredUserDomain - it shows that when using '?' for a key field, this one becomes null-aware, meaning a null value can be used as a join key"
+name = "sequenceWithOptionalUserDomain"
+expected_event_ids  = [1, 57]
+join_keys = ["null"]
+query = '''
+sequence by ?user_domain [process where true] [registry where true]
+'''
+
+[[queries]]
+name = "sequenceWithRequiredUserDomain"
+expected_event_ids  = []
+join_keys = []
+query = '''
+sequence by user_domain [process where true] [registry where true]
+'''
+
+[[queries]]
+description = "An equivalent (without optional keys) query can be found in test_queries.toml - twoSequencesWithTwoKeys"
+name = "twoSequencesWithTwoKeys_AndOptionals"
+query = '''
+sequence by ?x
+  [process where true]        by unique_pid,  process_path,        ?z
+  [process where opcode == 1] by unique_ppid, parent_process_path, ?w
+'''
+expected_event_ids  = [48, 53,
+                       53, 54,
+                       54, 56,
+                       97, 98]
+join_keys = ["null", "48", "C:\\Python27\\python.exe", "null",
+             "null", "53", "C:\\Windows\\System32\\cmd.exe", "null",
+             "null", "54", "C:\\Python27\\python.exe", "null",
+             "null", "750058", "C:\\Windows\\System32\\net.exe", "null"]
+
+[[queries]]
+name = "sequenceOnOneNullKey"
+query = '''
+sequence
+  [process where parent_process_path == null] by parent_process_path
+  [any where true] by parent_process_path
+'''
+expected_event_ids = []
+
+[[queries]]
+name = "sequenceOnTwoNullKeys"
+query = '''
+sequence by ppid 
+  [process where parent_process_path == null] by parent_process_path
+  [any where true] by parent_process_path
+'''
+expected_event_ids = []
+
+[[queries]]
+name = "sequenceOnImplicitNullKeys"
+query = '''
+sequence by ppid, parent_process_path
+  [process where parent_process_path == null]
+  [any where true]
+'''
+expected_event_ids = []

x-pack/plugin/eql/qa/common/src/main/resources/data/extra.data

@@ -21,18 +21,137 @@
   "@timestamp": "10",
   "event_type": "REQUEST",
   "transID": 1235,
-  "sequence": 1
+  "sequence": 4
  },
  {
   "@timestamp": "11",
   "event_type": "ERROR",
   "transID": 1235,
-  "sequence": 2
+  "sequence": 5
  },
  {
   "@timestamp": "11",
   "event_type": "STAT",
   "transID": 1235,
-  "sequence": 3
+  "sequence": 6
+ },
+ {
+  "@timestamp": "100",
+  "event_type": "OPTIONAL",
+  "optional_field_default_null": null,
+  "sequence": 7
+ },
+ {
+  "@timestamp": "101",
+  "event_type": "OPTIONAL",
+  "sequence": 8,
+  "transID": 1235
+ },
+ {
+  "@timestamp": "102",
+  "event_type": "OPTIONAL",
+  "optional_field_default_null": null,
+  "sequence": 9
+ },
+ {
+  "@timestamp": "1",
+  "event_type": "process",
+  "transID": 1,
+  "process.pid": 123,
+  "sequence": 10
+ },
+ {
+  "@timestamp": "1",
+  "event_type": "process",
+  "transID": 2,
+  "sequence": 11
+ },
+ {
+  "@timestamp": "2",
+  "event_type": "process",
+  "transID": 2,
+  "process.pid": 123,
+  "sequence": 12
+ },
+ {
+  "@timestamp": "3",
+  "event_type": "file",
+  "transID": 0,
+  "process.pid": 123,
+  "sequence": 13
+ },
+ {
+  "@timestamp": "4",
+  "event_type": "file",
+  "transID": 0,
+  "process.pid": 123,
+  "sequence": 14
+ },
+ {
+  "@timestamp": "5",
+  "event_type": "file",
+  "transID": 0,
+  "process.pid": 123,
+  "sequence": 15
+ },
+ {
+  "@timestamp": "6",
+  "event_type": "file",
+  "transID": 0,
+  "sequence": 16
+ },
+  {
+  "@timestamp": "6",
+  "event_type": "file",
+  "transID": 0,
+  "sequence": 17
+ },
+ {
+  "@timestamp": "7",
+  "event_type": "process",
+  "transID": 2,
+  "process.entity_id": 512,
+  "process.pid": 123,
+  "sequence": 18
+ },
+ {
+  "@timestamp": "8",
+  "event_type": "file",
+  "transID": 0,
+  "process.entity_id": 512,
+  "process.pid": 123,
+  "sequence": 19
+ },
+ {
+  "@timestamp": "9",
+  "event_type": "file",
+  "transID": 0,
+  "process.entity_id": 512,
+  "process.pid": 123,
+  "sequence": 20
+ },
+ {
+  "@timestamp": "10",
+  "event_type": "file",
+  "transID": 0,
+  "process.entity_id": 512,
+  "process.pid": 123,
+  "sequence": 21
+ },
+ {
+  "@timestamp": "11",
+  "event_type": "file",
+  "transID": 0,
+  "process.entity_id": 512,
+  "process.pid": 123,
+  "sequence": 22
+ },
+ {
+  "@timestamp": "12",
+  "event_type": "file",
+  "transID": 0,
+  "process.entity_id": 512,
+  "process.pid": 123,
+  "sequence": 23
  }
 ]

x-pack/plugin/eql/qa/common/src/main/resources/data/extra.mapping

@@ -20,6 +20,13 @@
           "path": "sequence"
         }
       }
+    },
+    "optional_field_mapping_only": {
+      "type": "keyword"
+    },
+    "optional_field_default_null": {
+      "type": "keyword",
+      "null_value": "NULL"
     }
   }
 }