Skip to content

Commit

Permalink
opensearch-project#683 Sigma keywords field not handled correctly (op…
Browse files Browse the repository at this point in the history 
…ensearch-project#725) (opensearch-project#739)

* changed windows sample rule and query construction

Signed-off-by: Joanne Wang <jowg@amazon.com>

* remove wildcard

Signed-off-by: Joanne Wang <jowg@amazon.com>

* changed wildcardtest

Signed-off-by: Joanne Wang <jowg@amazon.com>

* fixed wildcards

Signed-off-by: Joanne Wang <jowg@amazon.com>

* fixed wildcard query test

Signed-off-by: Joanne Wang <jowg@amazon.com>

* fixed correlation engine tests

Signed-off-by: Joanne Wang <jowg@amazon.com>

* fixed query backend tests

Signed-off-by: Joanne Wang <jowg@amazon.com>

* clean up

Signed-off-by: Joanne Wang <jowg@amazon.com>

* added two integration tests

Signed-off-by: Joanne Wang <jowg@amazon.com>

---------

Signed-off-by: Joanne Wang <jowg@amazon.com>
(cherry picked from commit 43040d6)

Co-authored-by: Joanne Wang <109310487+jowg-amazon@users.noreply.github.com>
  • Loading branch information
@opensearch-trigger-bot @jowg-amazon
opensearch-trigger-bot[bot] and jowg-amazon committed Jan 17, 2024
1 parent 4713ea0 commit 8990aed
Show file tree
Hide file tree
Showing 5 changed files with 451 additions and 23 deletions.
24 changes: 7 additions & 17 deletions src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,10 +132,10 @@ public OSQueryBackend(Map<String, String> fieldMappings, boolean collectErrors,
this.reExpression = "%s: /%s/";
this.cidrExpression = "%s: \"%s\"";
this.fieldNullExpression = "%s: null";
this.unboundValueStrExpression = "%s: \"%s\"";
this.unboundValueNumExpression = "%s: %s";
this.unboundWildcardExpression = "%s: %s";
this.unboundReExpression = "%s: /%s/";
this.unboundValueStrExpression = "\"%s\"";
this.unboundValueNumExpression = "\"%s\"";
this.unboundWildcardExpression = "%s";
this.unboundReExpression = "/%s/";
this.compareOpExpression = "\"%s\" \"%s\" %s";
this.valExpCount = 0;
this.aggQuery = "{\"%s\":{\"terms\":{\"field\":\"%s\"},\"aggs\":{\"%s\":{\"%s\":{\"field\":\"%s\"}}}}}";
Expand Down Expand Up @@ -332,28 +332,18 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre
@Override
public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError {
SigmaString value = (SigmaString) condition.getValue();

String field = getFinalValueField();
ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
boolean containsWildcard = value.containsWildcard();
return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), field, this.convertValueStr((SigmaString) condition.getValue()));
return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue()));
}

@Override
public Object convertConditionValNum(ConditionValueExpression condition) {
String field = getFinalValueField();

SigmaNumber number = (SigmaNumber) condition.getValue();
ruleQueryFields.put(field, number.getNumOpt().isLeft()? Collections.singletonMap("type", "integer"): Collections.singletonMap("type", "float"));

return String.format(Locale.getDefault(), this.unboundValueNumExpression, field, condition.getValue().toString());
return String.format(Locale.getDefault(), this.unboundValueNumExpression, condition.getValue().toString());
}

@Override
public Object convertConditionValRe(ConditionValueExpression condition) {
String field = getFinalValueField();
ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer"));
return String.format(Locale.getDefault(), this.unboundReExpression, field, convertValueRe((SigmaRegularExpression) condition.getValue()));
return String.format(Locale.getDefault(), this.unboundReExpression, convertValueRe((SigmaRegularExpression) condition.getValue()));
}

// TODO: below methods will be supported when Sigma Expand Modifier is supported.
Expand Down
4 changes: 3 additions & 1 deletion src/main/resources/rules/test_windows/win_sample_rule.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ detection:
EventID: 22
Message|contains: 'C:\\Program Files\\nxlog\\nxlog.exe'
HostName|startswith: 'EC2AMAZ'
condition: selection
keywords:
- "NT AUTHORITY"
condition: selection or keywords
falsepositives:
- Unknown
160 changes: 159 additions & 1 deletion src/test/java/org/opensearch/securityanalytics/TestHelpers.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,100 @@ public static String randomRuleWithAlias() {
"level: high";
}

public static String randomRuleWithKeywords() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 21\n" +
" keywords:\n" +
" - 1996\n" +
" - EC2AMAZ*\n" +
" condition: selection or keywords\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithStringKeywords() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 21\n" +
" keywords:\n" +
" - \"INFO\"\n" +
" condition: selection or keywords\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String randomRuleWithDateKeywords() {
return "title: Remote Encrypting File System Abuse\n" +
"id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" +
"description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" +
"references:\n" +
" - https://attack.mitre.org/tactics/TA0008/\n" +
" - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" +
" - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" +
" - https://github.com/zeronetworks/rpcfirewall\n" +
" - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" +
"tags:\n" +
" - attack.defense_evasion\n" +
"status: experimental\n" +
"author: Sagie Dulce, Dekel Paz\n" +
"date: 2022/01/01\n" +
"modified: 2022/01/01\n" +
"logsource:\n" +
" product: rpc_firewall\n" +
" category: application\n" +
" definition: 'Requirements: install and apply the RPC Firewall to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" +
"detection:\n" +
" selection:\n" +
" EventID: 21\n" +
" keywords:\n" +
" - \"2020-02-04T14:59:39.343541+00:00\"\n" +
" condition: selection or keywords\n" +
"falsepositives:\n" +
" - Legitimate usage of remote file encryption\n" +
"level: high";
}

public static String countAggregationTestRule() {
return " title: Test\n" +
" id: 39f919f3-980b-4e6f-a975-8af7e507ef2b\n" +
Expand Down Expand Up @@ -1367,6 +1461,48 @@ public static String windowsIndexMapping() {
" }";
}

public static String windowsIndexMappingOnlyNumericAndDate() {
return "\"properties\": {\n" +
" \"@timestamp\": {\"type\":\"date\"},\n" +
" \"EventTime\": {\n" +
" \"type\": \"date\"\n" +
" },\n" +
" \"ExecutionProcessID\": {\n" +
" \"type\": \"long\"\n" +
" },\n" +
" \"ExecutionThreadID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"EventID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"TaskValue\": {\n" +
" \"type\": \"integer\"\n" +
" }\n" +
" }";
}

public static String windowsIndexMappingOnlyNumericAndText() {
return "\"properties\": {\n" +
" \"TaskName\": {\n" +
" \"type\": \"text\"\n" +
" },\n" +
" \"ExecutionProcessID\": {\n" +
" \"type\": \"long\"\n" +
" },\n" +
" \"ExecutionThreadID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"EventID\": {\n" +
" \"type\": \"integer\"\n" +
" },\n" +
" \"TaskValue\": {\n" +
" \"type\": \"integer\"\n" +
" }\n" +
" }";
}


public static String randomDoc(int severity, int version, String opCode) {
String doc = "{\n" +
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
Expand Down Expand Up @@ -1406,6 +1542,28 @@ public static String randomDoc(int severity, int version, String opCode) {

}

public static String randomDocOnlyNumericAndDate(int severity, int version, String opCode) {
String doc = "{\n" +
"\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" +
"\"ExecutionProcessID\":2001,\n" +
"\"ExecutionThreadID\":2616,\n" +
"\"EventID\": 1234,\n" +
"\"TaskValue\":22\n" +
"}";
return String.format(Locale.ROOT, doc, severity, version, opCode);
}

public static String randomDocOnlyNumericAndText(int severity, int version, String opCode) {
String doc = "{\n" +
"\"TaskName\":\"SYSTEM\",\n" +
"\"ExecutionProcessID\":2001,\n" +
"\"ExecutionThreadID\":2616,\n" +
"\"EventID\": 1234,\n" +
"\"TaskValue\":22\n" +
"}";
return String.format(Locale.ROOT, doc, severity, version, opCode);
}

//Add IPs in HostName field.
public static String randomDocWithIpIoc(int severity, int version, String ioc) {
String doc = "{\n" +
Expand Down Expand Up @@ -1510,7 +1668,7 @@ public static String randomAppLogDoc() {
return "{\n" +
" \"endpoint\": \"/customer_records.txt\",\n" +
" \"http_method\": \"POST\",\n" +
" \"keywords\": \"PermissionDenied\"\n" +
" \"keywords\": \"INVALID\"\n" +
"}";
}

Expand Down
Loading

0 comments on commit 8990aed

Please sign in to comment.