From 04423efc86e7f2628d072b5a1d5c97fb95e52745 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Tue, 21 Feb 2023 01:01:58 +0100 Subject: [PATCH 01/11] query_field_names bugfix (#335) Signed-off-by: Petar Dzepina --- .../securityanalytics/mapper/MapperService.java | 7 +++++++ .../rules/backend/QueryBackend.java | 4 ++++ .../securityanalytics/util/RuleIndices.java | 1 + src/main/resources/mappings/rules.json | 2 +- .../securityanalytics/mapper/MapperRestApiIT.java | 14 +++++++------- 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index ff7479372..079984d36 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -292,6 +292,13 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { } } + if (appliedAliases.size() == 0) { + actionListener.onFailure(SecurityAnalyticsException.wrap( + new OpenSearchStatusException("No applied aliases found", RestStatus.NOT_FOUND)) + ); + return; + } + // Traverse mappings and do copy with excluded type=alias properties MappingsTraverser mappingsTraverser = new MappingsTraverser(mappingMetadata); // Resulting mapping after filtering diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java index c6e740c3f..bf2a4117b 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/QueryBackend.java @@ -180,6 +180,10 @@ public Map getQueryFields() { return queryFields; } + public void resetQueryFields() { + queryFields.clear(); + } + public abstract Object convertConditionAsInExpression(Either condition); public abstract Object convertConditionAnd(ConditionAND condition); diff --git a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java index f6b7b7247..7f89cc57b 100644 --- a/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java +++ b/src/main/java/org/opensearch/securityanalytics/util/RuleIndices.java @@ -289,6 +289,7 @@ private List getQueries(QueryBackend backend, String category, List queries = new ArrayList<>(); for (String ruleStr: rules) { SigmaRule rule = SigmaRule.fromYaml(ruleStr, true); + backend.resetQueryFields(); List ruleQueries = backend.convertRule(rule); Set queryFieldNames = backend.getQueryFields().keySet(); diff --git a/src/main/resources/mappings/rules.json b/src/main/resources/mappings/rules.json index 39b9621d0..397331805 100644 --- a/src/main/resources/mappings/rules.json +++ b/src/main/resources/mappings/rules.json @@ -100,7 +100,7 @@ } } }, - "query_fields": { + "query_field_names": { "type": "nested", "properties": { "value": { diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 992166bb2..a73887671 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -270,13 +270,13 @@ public void testUpdateAndGetMapping_notFound_Success() throws IOException { // Execute GetIndexMappingsAction and verify mappings Request getRequest = new Request("GET", SecurityAnalyticsPlugin.MAPPER_BASE_URI); getRequest.addParameter("index_name", testIndexName); - response = client().performRequest(getRequest); - XContentParser parser = createParser(JsonXContent.jsonXContent, new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8)); - assertTrue( - (((Map)((Map)parser.map() - .get(testIndexName)) - .get("mappings")) - .containsKey("properties"))); + try { + client().performRequest(getRequest); + fail(); + } catch (ResponseException e) { + assertEquals(HttpStatus.SC_NOT_FOUND, e.getResponse().getStatusLine().getStatusCode()); + assertTrue(e.getMessage().contains("No applied aliases found")); + } } public void testExistingMappingsAreUntouched() throws IOException { From fbc04163f4393e7e5694147aafd3a377f6443053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrico=20Tr=C3=B6ger?= Date: Wed, 12 Jul 2023 19:18:26 +0200 Subject: [PATCH 02/11] Reduce log level for informative message (#203) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Enrico Tröger --- .../indexmanagment/DetectorIndexManagementService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java index 98fdc08e9..eec89d19e 100644 --- a/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java +++ b/src/main/java/org/opensearch/securityanalytics/indexmanagment/DetectorIndexManagementService.java @@ -233,7 +233,7 @@ private String executorName() { } private void deleteOldIndices(String tag, String... indices) { - logger.error("info deleteOldIndices"); + logger.info("info deleteOldIndices"); ClusterStateRequest clusterStateRequest = new ClusterStateRequest() .clear() .indices(indices) From 7357bb484b659b583ce2dcc1c227e4995476729c Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Thu, 26 Oct 2023 09:33:50 -0700 Subject: [PATCH 03/11] fix detector writeTo() method missing fields (#695) * fix detector writeTo() method missing fields Signed-off-by: Surya Sashank Nistala * fix test Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala --- .../securityanalytics/model/Detector.java | 20 ++++++++------ .../model/WriteableTests.java | 27 +++++++++++++++++++ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/model/Detector.java b/src/main/java/org/opensearch/securityanalytics/model/Detector.java index a05f04b81..aec4480dc 100644 --- a/src/main/java/org/opensearch/securityanalytics/model/Detector.java +++ b/src/main/java/org/opensearch/securityanalytics/model/Detector.java @@ -159,12 +159,12 @@ public Detector(StreamInput sin) throws IOException { sin.readList(DetectorInput::readFrom), sin.readList(DetectorTrigger::readFrom), sin.readStringList(), - sin.readString(), - sin.readString(), - sin.readString(), - sin.readString(), - sin.readString(), - sin.readString(), + sin.readOptionalString(), + sin.readOptionalString(), + sin.readOptionalString(), + sin.readOptionalString(), + sin.readOptionalString(), + sin.readOptionalString(), sin.readMap(StreamInput::readString, StreamInput::readString) ); } @@ -197,8 +197,12 @@ public void writeTo(StreamOutput out) throws IOException { it.writeTo(out); } out.writeStringCollection(monitorIds); - out.writeString(ruleIndex); - + out.writeOptionalString(ruleIndex); + out.writeOptionalString(alertsIndex); + out.writeOptionalString(alertsHistoryIndex); + out.writeOptionalString(alertsHistoryIndexPattern); + out.writeOptionalString(findingsIndex); + out.writeOptionalString(findingsIndexPattern); out.writeMap(ruleIdMonitorIdMap, StreamOutput::writeString, StreamOutput::writeString); } diff --git a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java index 2326b541d..01f1cea70 100644 --- a/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java +++ b/src/test/java/org/opensearch/securityanalytics/model/WriteableTests.java @@ -13,15 +13,42 @@ import java.io.IOException; import java.util.List; +import static org.opensearch.securityanalytics.TestHelpers.parser; import static org.opensearch.securityanalytics.TestHelpers.randomDetector; import static org.opensearch.securityanalytics.TestHelpers.randomUser; import static org.opensearch.securityanalytics.TestHelpers.randomUserEmpty; +import static org.opensearch.securityanalytics.TestHelpers.toJsonStringWithUser; public class WriteableTests extends OpenSearchTestCase { public void testDetectorAsStream() throws IOException { Detector detector = randomDetector(List.of()); detector.setInputs(List.of(new DetectorInput("", List.of(), List.of(), List.of()))); + logger.error(toJsonStringWithUser(detector)); + BytesStreamOutput out = new BytesStreamOutput(); + detector.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + Detector newDetector = new Detector(sin); + Assert.assertEquals("Round tripping Detector doesn't work", detector, newDetector); + } + + public void testDetector() throws IOException { // an edge case of detector serialization that failed testDetectorAsAStream() intermittently + String detectorString = "{\"type\":\"detector\",\"name\":\"MczAuRCrve\",\"detector_type\":\"test_windows\"," + + "\"user\":{\"name\":\"QhKrfthgxw\",\"backend_roles\":[\"uYvGLCPhfX\",\"fOLkcRxMWR\"],\"roles\"" + + ":[\"YuucNpVzTm\",\"all_access\"],\"custom_attribute_names\":[\"test_attr=test\"]," + + "\"user_requested_tenant\":null},\"threat_intel_enabled\":false,\"enabled\":false,\"enabled_time\"" + + ":null,\"schedule\":{\"period\":{\"interval\":5,\"unit\":\"MINUTES\"}},\"inputs\":[{\"detector_input\"" + + ":{\"description\":\"\",\"indices\":[],\"custom_rules\":[],\"pre_packaged_rules\":[]}}],\"triggers\"" + + ":[{\"id\":\"SiWfaosBBiNA8if0E1bC\",\"name\":\"windows-trigger\",\"severity\":\"1\",\"types\"" + + ":[\"test_windows\"],\"ids\":[\"QuarksPwDump Clearing Access History\"],\"sev_levels\":[\"high\"]," + + "\"tags\":[\"T0008\"],\"actions\":[],\"detection_types\":[\"rules\"]}],\"last_update_time\":" + + "1698300892093,\"monitor_id\":[\"\"],\"workflow_ids\":[],\"bucket_monitor_id_rule_id\"" + + ":{},\"rule_topic_index\":\"\",\"alert_index\":\"\",\"alert_history_index\":\"\"," + + "\"alert_history_index_pattern\":\"\",\"findings_index\":\"\",\"findings_index_pattern\":\"\"}"; + Detector detector = Detector.parse(parser(detectorString), null, null); +// Detector detector = randomDetector(List.of()); +// detector.setInputs(List.of(new DetectorInput("", List.of(), List.of(), List.of()))); +// logger.error(toJsonStringWithUser(detector)); BytesStreamOutput out = new BytesStreamOutput(); detector.writeTo(out); StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); From 1e9d5d896a3a14d3192b7189c62e03e1c5d25979 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Wed, 29 Nov 2023 15:59:47 -0800 Subject: [PATCH 04/11] fix null query filter conversion from sigma to query string query (#722) * fix null query filter conversion from sigma to query string query Signed-off-by: Surya Sashank Nistala * fix rule to query conversion tests for null filter Signed-off-by: Surya Sashank Nistala * enhance test to verify non null doc doesnt match null query Signed-off-by: Surya Sashank Nistala --------- Signed-off-by: Surya Sashank Nistala --- .../rules/backend/OSQueryBackend.java | 2 +- .../securityanalytics/TestHelpers.java | 69 +++++++++++++++++++ .../rules/backend/QueryBackendTests.java | 4 +- 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index b1da78b5c..ca035b648 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -131,7 +131,7 @@ public OSQueryBackend(String ruleCategory, boolean collectErrors, boolean enable this.reEscapeChar = "\\"; this.reExpression = "%s: /%s/"; this.cidrExpression = "%s: \"%s\""; - this.fieldNullExpression = "%s: null"; + this.fieldNullExpression = "%s: (NOT [* TO *])"; this.unboundValueStrExpression = "\"%s\""; this.unboundValueNumExpression = "\"%s\""; this.unboundWildcardExpression = "%s"; diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 29a2b040e..34412b806 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -168,6 +168,37 @@ public static String randomRule() { "level: high"; } + + public static String randomNullRule() { + return "title: null field\n" + + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + + "description: Detects remote RPC calls to possibly abuse remote encryption service via MS-EFSR\n" + + "references:\n" + + " - https://attack.mitre.org/tactics/TA0008/\n" + + " - https://msrc.microsoft.com/update-guide/vulnerability/CVE-2021-36942\n" + + " - https://github.com/jsecurity101/MSRPC-to-ATTACK/blob/main/documents/MS-EFSR.md\n" + + " - https://github.com/zeronetworks/rpcfirewall\n" + + " - https://zeronetworks.com/blog/stopping_lateral_movement_via_the_rpc_firewall/\n" + + "tags:\n" + + " - attack.defense_evasion\n" + + "status: experimental\n" + + "author: Sagie Dulce, Dekel Paz\n" + + "date: 2022/01/01\n" + + "modified: 2022/01/01\n" + + "logsource:\n" + + " product: rpc_firewall\n" + + " category: application\n" + + " definition: 'Requirements: install and apply the RPC Firew all to all processes with \"audit:true action:block uuid:df1941c5-fe89-4e79-bf10-463657acf44d or c681d488-d850-11d0-8c52-00c04fd90f7e'\n" + + "detection:\n" + + " selection:\n" + + " EventID: 22\n" + + " RecordNumber: null\n" + + " condition: selection\n" + + "falsepositives:\n" + + " - Legitimate usage of remote file encryption\n" + + "level: high"; + } + public static String randomRuleWithKeywords() { return "title: Remote Encrypting File System Abuse\n" + "id: 5f92fff9-82e2-48eb-8fc1-8b133556a551\n" + @@ -1353,6 +1384,44 @@ public static String randomDocOnlyNumericAndText(int severity, int version, Stri return String.format(Locale.ROOT, doc, severity, version, opCode); } + public static String randomDocWithNullField() { + return "{\n" + + "\"@timestamp\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + + "\"HostName\":\"EC2AMAZ-EPO7HKA\",\n" + + "\"Keywords\":\"9223372036854775808\",\n" + + "\"SeverityValue\":2,\n" + + "\"Severity\":\"INFO\",\n" + + "\"EventID\":22,\n" + + "\"SourceName\":\"Microsoft-Windows-Sysmon\",\n" + + "\"ProviderGuid\":\"{5770385F-C22A-43E0-BF4C-06F5698FFBD9}\",\n" + + "\"Version\":5,\n" + + "\"TaskValue\":22,\n" + + "\"OpcodeValue\":0,\n" + + "\"RecordNumber\":null,\n" + + "\"ExecutionProcessID\":1996,\n" + + "\"ExecutionThreadID\":2616,\n" + + "\"Channel\":\"Microsoft-Windows-Sysmon/Operational\",\n" + + "\"Domain\":\"NTAUTHORITY\",\n" + + "\"AccountName\":\"SYSTEM\",\n" + + "\"UserID\":\"S-1-5-18\",\n" + + "\"AccountType\":\"User\",\n" + + "\"Message\":\"Dns query:\\r\\nRuleName: \\r\\nUtcTime: 2020-02-04 14:59:38.349\\r\\nProcessGuid: {b3c285a4-3cda-5dc0-0000-001077270b00}\\r\\nProcessId: 1904\\r\\nQueryName: EC2AMAZ-EPO7HKA\\r\\nQueryStatus: 0\\r\\nQueryResults: 172.31.46.38;\\r\\nImage: C:\\\\Program Files\\\\nxlog\\\\nxlog.exe\",\n" + + "\"Category\":\"Dns query (rule: DnsQuery)\",\n" + + "\"Opcode\":\"Info\",\n" + + "\"UtcTime\":\"2020-02-04 14:59:38.349\",\n" + + "\"ProcessGuid\":\"{b3c285a4-3cda-5dc0-0000-001077270b00}\",\n" + + "\"ProcessId\":\"1904\",\"QueryName\":\"EC2AMAZ-EPO7HKA\",\"QueryStatus\":\"0\",\n" + + "\"QueryResults\":\"172.31.46.38;\",\n" + + "\"Image\":\"C:\\\\Program Files\\\\nxlog\\\\regsvr32.exe\",\n" + + "\"EventReceivedTime\":\"2020-02-04T14:59:40.780905+00:00\",\n" + + "\"SourceModuleName\":\"in\",\n" + + "\"SourceModuleType\":\"im_msvistalog\",\n" + + "\"CommandLine\": \"eachtest\",\n" + + "\"Initiated\": \"true\"\n" + + "}"; + } + public static String randomDoc() { return "{\n" + "\"EventTime\":\"2020-02-04T14:59:39.343541+00:00\",\n" + diff --git a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java index c5917cf8b..54d5ff01c 100644 --- a/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java +++ b/src/test/java/org/opensearch/securityanalytics/rules/backend/QueryBackendTests.java @@ -282,7 +282,7 @@ public void testConvertValueNull() throws IOException, SigmaError { " sel:\n" + " fieldA1: null\n" + " condition: sel", false)); - Assert.assertEquals("mappedA: null", queries.get(0).toString()); + Assert.assertEquals("mappedA: (NOT [* TO *])", queries.get(0).toString()); } public void testConvertValueRegex() throws IOException, SigmaError { @@ -525,7 +525,7 @@ public void testConvertOrInUnallowedValueType() throws IOException, SigmaError { " - value2\n" + " - null\n" + " condition: sel", false)); - Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: null)", queries.get(0).toString()); + Assert.assertEquals("(mappedA: \"value1\") OR (mappedA: \"value2\") OR (mappedA: (NOT [* TO *]))", queries.get(0).toString()); } public void testConvertOrInListNumbers() throws IOException, SigmaError { From 82f97a58026edb5d371280c02ba46b569352220b Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 14 Mar 2024 13:19:46 -0700 Subject: [PATCH 05/11] Bump version 2.5.1, fix build Signed-off-by: Chase Engelbrecht --- build.gradle | 62 +++++++++++++++------------------------------------- 1 file changed, 18 insertions(+), 44 deletions(-) diff --git a/build.gradle b/build.gradle index 48984a3f9..3de8e5e1c 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ import org.opensearch.gradle.test.RestIntegTestTask buildscript { ext { - opensearch_version = System.getProperty("opensearch.version", "2.5.0-SNAPSHOT") + opensearch_version = System.getProperty("opensearch.version", "2.5.1-SNAPSHOT") isSnapshot = "true" == System.getProperty("build.snapshot", "true") buildVersionQualifier = System.getProperty("build.version_qualifier", "") version_tokens = opensearch_version.tokenize('-') @@ -125,6 +125,9 @@ repositories { } sourceSets.main.java.srcDirs = ['src/main/generated','src/main/java'] +configurations { + zipArchive +} dependencies { javaRestTestImplementation project.sourceSets.main.runtimeClasspath @@ -134,6 +137,11 @@ dependencies { api "org.opensearch:common-utils:${common_utils_version}" api "org.opensearch.client:opensearch-rest-client:${opensearch_version}" implementation "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}" + + // Needed for integ tests + zipArchive group: 'org.opensearch.plugin', name:'alerting', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'opensearch-notifications-core', version: "${opensearch_build}" + zipArchive group: 'org.opensearch.plugin', name:'notifications', version: "${opensearch_build}" } // RPM & Debian build @@ -209,15 +217,6 @@ integTest.getClusters().forEach{c -> { c.plugin(project.getObjects().fileProperty().value(bundle.getArchiveFile())) }} -String alertingFilePath = "src/test/resources/alerting" -String alertingPlugin = "opensearch-alerting-" + plugin_no_snapshot + ".zip" -String alertingRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + alertingPlugin -String notificationsFilePath = "src/test/resources/notifications" -String notificationsCoreFilePath = "src/test/resources/notifications-core" -String notificationsPlugin = "opensearch-notifications-" + plugin_no_snapshot + ".zip" -String notificationsCorePlugin = "opensearch-notifications-core-" + plugin_no_snapshot + ".zip" -String notificationsRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsPlugin -String notificationsCoreRemoteFile = "https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/" + opensearch_no_snapshot + "/latest/linux/x64/tar/builds/opensearch/plugins/" + notificationsCorePlugin testClusters.integTest { testDistribution = 'ARCHIVE' @@ -233,22 +232,13 @@ testClusters.integTest { debugPort += 1 } } - setting 'path.repo', repo.absolutePath plugin(provider({ new RegularFile() { @Override File getAsFile() { - File dir = new File(rootDir.path + "/" + alertingFilePath) - - if (!dir.exists()) { - dir.mkdirs() - } - - File f = new File(dir, alertingPlugin) - if (!f.exists()) { - new URL(alertingRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }} - } - fileTree(alertingFilePath).getSingleFile() + return configurations.zipArchive.asFileTree.matching { + include '**/opensearch-notifications-core*' + }.singleFile } } })) @@ -256,17 +246,9 @@ testClusters.integTest { new RegularFile() { @Override File getAsFile() { - File dir = new File(rootDir.path + "/" + notificationsCoreFilePath) - - if (!dir.exists()) { - dir.mkdirs() - } - - File f = new File(dir, notificationsCorePlugin) - if (!f.exists()) { - new URL(notificationsCoreRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }} - } - fileTree(notificationsCoreFilePath).getSingleFile() + return configurations.zipArchive.asFileTree.matching { + include '**/notifications*' + }.singleFile } } })) @@ -274,17 +256,9 @@ testClusters.integTest { new RegularFile() { @Override File getAsFile() { - File dir = new File(rootDir.path + "/" + notificationsFilePath) - - if (!dir.exists()) { - dir.mkdirs() - } - - File f = new File(dir, notificationsPlugin) - if (!f.exists()) { - new URL(notificationsRemoteFile).withInputStream{ ins -> f.withOutputStream{ it << ins }} - } - fileTree(notificationsFilePath).getSingleFile() + return configurations.zipArchive.asFileTree.matching { + include '**/alerting*' + }.singleFile } } })) From f2a78b5c77989a93749ed429c004f6ce923437bf Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 14 Mar 2024 14:56:43 -0700 Subject: [PATCH 06/11] Manual cherry-pick of #873 Signed-off-by: Chase Engelbrecht --- .../mapper/MapperService.java | 6 +- .../rules/backend/OSQueryBackend.java | 2 + .../TransportIndexDetectorAction.java | 514 +++++++++++++----- 3 files changed, 372 insertions(+), 150 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 079984d36..882d056ba 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -71,9 +71,11 @@ public void createMappingAction(String indexName, String ruleTopic, String alias // since you can't update documents in non-write indices String index = indexName; boolean shouldUpsertIndexTemplate = IndexUtils.isConcreteIndex(indexName, this.clusterService.state()) == false; - if (IndexUtils.isDataStream(indexName, this.clusterService.state())) { + if (IndexUtils.isDataStream(indexName, this.clusterService.state()) || IndexUtils.isAlias(indexName, this.clusterService.state())) { + log.debug("{} is an alias or datastream. Fetching write index for create mapping action.", indexName); String writeIndex = IndexUtils.getWriteIndex(indexName, this.clusterService.state()); if (writeIndex != null) { + log.debug("Write index for {} is {}", indexName, writeIndex); index = writeIndex; } } @@ -85,6 +87,7 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { applyAliasMappings(getMappingsResponse.getMappings(), ruleTopic, aliasMappings, partial, new ActionListener<>() { @Override public void onResponse(Collection createMappingResponse) { + log.debug("Completed create mappings for {}", indexName); // We will return ack==false if one of the requests returned that // else return ack==true Optional notAckd = createMappingResponse.stream() @@ -103,6 +106,7 @@ public void onResponse(Collection createMappingResponse) { @Override public void onFailure(Exception e) { + log.debug("Failed to create mappings for {}", indexName ); actionListener.onFailure(e); } }); diff --git a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java index ca035b648..a6b2dc7ae 100644 --- a/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java +++ b/src/main/java/org/opensearch/securityanalytics/rules/backend/OSQueryBackend.java @@ -331,6 +331,8 @@ public Object convertConditionFieldEqValQueryExpr(ConditionFieldEqualsValueExpre @Override public Object convertConditionValStr(ConditionValueExpression condition) throws SigmaValueError { + String field = getFinalValueField(); + ruleQueryFields.put(field, Map.of("type", "text", "analyzer", "rule_analyzer")); SigmaString value = (SigmaString) condition.getValue(); boolean containsWildcard = value.containsWildcard(); return String.format(Locale.getDefault(), (containsWildcard? this.unboundWildcardExpression: this.unboundValueStrExpression), this.convertValueStr((SigmaString) condition.getValue())); diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index d7b7ae879..e3a779d57 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -107,9 +107,11 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; @@ -197,17 +199,21 @@ private void checkIndicesAndExecute( ActionListener listener, User user ) { + log.debug("check indices and execute began"); String [] detectorIndices = request.getDetector().getInputs().stream().flatMap(detectorInput -> detectorInput.getIndices().stream()).toArray(String[]::new); - SearchRequest searchRequest = new SearchRequest(detectorIndices).source(SearchSourceBuilder.searchSource().size(1).query(QueryBuilders.matchAllQuery()));; + SearchRequest searchRequest = new SearchRequest(detectorIndices).source(SearchSourceBuilder.searchSource().size(1).query(QueryBuilders.matchAllQuery())); + searchRequest.setCancelAfterTimeInterval(TimeValue.timeValueSeconds(30)); client.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse searchResponse) { + log.debug("check indices and execute completed. Took {} millis", searchResponse.getTook().millis()); AsyncIndexDetectorsAction asyncAction = new AsyncIndexDetectorsAction(user, task, request, listener); asyncAction.start(); } @Override public void onFailure(Exception e) { + log.debug("check indices and execute failed", e); if (e instanceof OpenSearchStatusException) { listener.onFailure(SecurityAnalyticsException.wrap( new OpenSearchStatusException(String.format(Locale.getDefault(), "User doesn't have read permissions for one or more configured index %s", detectorIndices), RestStatus.FORBIDDEN) @@ -224,61 +230,85 @@ public void onFailure(Exception e) { }); } - private void createMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy) throws SigmaError, IOException { + private void createMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy, + List queryFieldNames) { List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( Collectors.toList()); List> bucketLevelRules = rulesById.stream().filter(it -> it.getRight().isAggregationRule()).collect( Collectors.toList()); - List monitorRequests = new ArrayList<>(); + try { + List monitorRequests = new ArrayList<>(); - if (!docLevelRules.isEmpty()) { - monitorRequests.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); - } - if (!bucketLevelRules.isEmpty()) { - monitorRequests.addAll(buildBucketLevelMonitorRequests(Pair.of(index, bucketLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); - } - // Do nothing if detector doesn't have any monitor - if (monitorRequests.isEmpty()){ - listener.onResponse(Collections.emptyList()); - return; - } + if (!docLevelRules.isEmpty()) { + monitorRequests.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST, queryFieldNames)); + } + if (!bucketLevelRules.isEmpty()) { + StepListener> bucketLevelMonitorRequests = new StepListener<>(); + buildBucketLevelMonitorRequests(Pair.of(index, bucketLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST, bucketLevelMonitorRequests); + bucketLevelMonitorRequests.whenComplete(indexMonitorRequests -> { + log.debug("bucket level monitor request built"); + monitorRequests.addAll(indexMonitorRequests); + + // Do nothing if detector doesn't have any monitor + if (monitorRequests.isEmpty()) { + listener.onResponse(Collections.emptyList()); + return; + } - List monitorResponses = new ArrayList<>(); - StepListener addFirstMonitorStep = new StepListener(); - - // Indexing monitors in two steps in order to prevent all shards failed error from alerting - // https://github.com/opensearch-project/alerting/issues/646 - AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, addFirstMonitorStep); - addFirstMonitorStep.whenComplete(addedFirstMonitorResponse -> { - monitorResponses.add(addedFirstMonitorResponse); - int numberOfUnprocessedResponses = monitorRequests.size() - 1; - if (numberOfUnprocessedResponses == 0){ - listener.onResponse(monitorResponses); - } else { - GroupedActionListener monitorResponseListener = new GroupedActionListener( - new ActionListener>() { - @Override - public void onResponse(Collection indexMonitorResponse) { - monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); - listener.onResponse(monitorResponses); - } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }, numberOfUnprocessedResponses); + List monitorResponses = new ArrayList<>(); + StepListener addFirstMonitorStep = new StepListener(); + + // Indexing monitors in two steps in order to prevent all shards failed error from alerting + // https://github.com/opensearch-project/alerting/issues/646 + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, addFirstMonitorStep); + addFirstMonitorStep.whenComplete(addedFirstMonitorResponse -> { + log.debug("first monitor created id {} of type {}", addedFirstMonitorResponse.getId(), addedFirstMonitorResponse.getMonitor().getMonitorType()); + monitorResponses.add(addedFirstMonitorResponse); + int numberOfUnprocessedResponses = monitorRequests.size() - 1; + if (numberOfUnprocessedResponses == 0) { + listener.onResponse(monitorResponses); + } else { + GroupedActionListener monitorResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection indexMonitorResponse) { + monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); + listener.onResponse(monitorResponses); + } - for(int i = 1; i < monitorRequests.size(); i++){ - AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); - } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, numberOfUnprocessedResponses); + + for (int i = 1; i < monitorRequests.size(); i++) { + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); + } + } + }, listener::onFailure); + }, listener::onFailure); + } else { + // Failure if detector doesn't have any monitor + if (monitorRequests.isEmpty()) { + listener.onFailure(new OpenSearchStatusException("Detector cannot be created as no compatible rules were provided", RestStatus.BAD_REQUEST)); + return; } - }, - listener::onFailure - ); + List monitorResponses = new ArrayList<>(); + StepListener indexDocLevelMonitorStep = new StepListener(); + // Indexing monitors in two steps in order to prevent all shards failed error from alerting + // https://github.com/opensearch-project/alerting/issues/646 + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, indexDocLevelMonitorStep); + indexDocLevelMonitorStep.whenComplete(monitorResponses::add, listener::onFailure); + } + } catch (Exception ex) { + listener.onFailure(ex); + } } - private void updateMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy) throws SigmaError, IOException { + private void updateMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy, + List queryFieldNames) throws IOException { List monitorsToBeUpdated = new ArrayList<>(); List> bucketLevelRules = rulesById.stream().filter(it -> it.getRight().isAggregationRule()).collect( @@ -295,6 +325,28 @@ private void updateMonitorFromQueries(String index, List> rul // Pair of RuleId - MonitorId for existing monitors of the detector Map monitorPerRule = detector.getRuleIdMonitorIdMap(); + GroupedActionListener groupedActionListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection indexMonitorRequests) { + onIndexMonitorRequestCreation( + index, + monitorsToBeUpdated, + monitorsToBeAdded, + rulesById, + detector, + refreshPolicy, + queryFieldNames, + listener + ); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, bucketLevelRules.size() + ); for (Pair query: bucketLevelRules) { Rule rule = query.getRight(); @@ -302,41 +354,77 @@ private void updateMonitorFromQueries(String index, List> rul // Detect if the monitor should be added or updated if (monitorPerRule.containsKey(rule.getId())) { String monitorId = monitorPerRule.get(rule.getId()); - monitorsToBeUpdated.add(createBucketLevelMonitorRequest(query.getRight(), + createBucketLevelMonitorRequest(query.getRight(), index, detector, refreshPolicy, monitorId, Method.PUT, - queryBackendMap.get(rule.getCategory()))); + queryBackendMap.get(rule.getCategory()), + new ActionListener<>() { + @Override + public void onResponse(IndexMonitorRequest indexMonitorRequest) { + monitorsToBeUpdated.add(indexMonitorRequest); + groupedActionListener.onResponse(indexMonitorRequest); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to create bucket level monitor request", e); + listener.onFailure(e); + } + + }); } else { - monitorsToBeAdded.add(createBucketLevelMonitorRequest(query.getRight(), + createBucketLevelMonitorRequest(query.getRight(), index, detector, refreshPolicy, Monitor.NO_ID, Method.POST, - queryBackendMap.get(rule.getCategory()))); + queryBackendMap.get(rule.getCategory()), + new ActionListener<>() { + @Override + public void onResponse(IndexMonitorRequest indexMonitorRequest) { + monitorsToBeAdded.add(indexMonitorRequest); + groupedActionListener.onResponse(indexMonitorRequest); + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to create bucket level monitor request", e); + listener.onFailure(e); + } + }); } } } } + } + private void onIndexMonitorRequestCreation(String index, + List monitorsToBeUpdated, + List monitorsToBeAdded, + List> rulesById, + Detector detector, + RefreshPolicy refreshPolicy, + List queryFieldNames, + ActionListener> listener) { List> docLevelRules = rulesById.stream().filter(it -> !it.getRight().isAggregationRule()).collect( - Collectors.toList()); + Collectors.toList()); // Process doc level monitors if (!docLevelRules.isEmpty()) { if (detector.getDocLevelMonitorId() == null) { - monitorsToBeAdded.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST)); + monitorsToBeAdded.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, Monitor.NO_ID, Method.POST, queryFieldNames)); } else { - monitorsToBeUpdated.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT)); + monitorsToBeUpdated.add(createDocLevelMonitorRequest(Pair.of(index, docLevelRules), detector, refreshPolicy, detector.getDocLevelMonitorId(), Method.PUT, queryFieldNames)); } } List monitorIdsToBeDeleted = detector.getRuleIdMonitorIdMap().values().stream().collect(Collectors.toList()); monitorIdsToBeDeleted.removeAll(monitorsToBeUpdated.stream().map(IndexMonitorRequest::getMonitorId).collect( - Collectors.toList())); + Collectors.toList())); updateAlertingMonitors(monitorsToBeAdded, monitorsToBeUpdated, monitorIdsToBeDeleted, refreshPolicy, listener); } @@ -394,7 +482,8 @@ private void updateAlertingMonitors( }, listener::onFailure); } - private IndexMonitorRequest createDocLevelMonitorRequest(Pair>> logIndexToQueries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) { + private IndexMonitorRequest createDocLevelMonitorRequest(Pair>> logIndexToQueries, Detector detector, + WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod, List queryFieldNames) { List docLevelMonitorInputs = new ArrayList<>(); List docLevelQueries = new ArrayList<>(); @@ -412,7 +501,7 @@ private IndexMonitorRequest createDocLevelMonitorRequest(Pair buildBucketLevelMonitorRequests(Pair>> logIndexToQueries, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod) throws IOException, SigmaError { + private void buildBucketLevelMonitorRequests(Pair>> logIndexToQueries, Detector detector, + WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod, + ActionListener> listener) { + log.debug("bucket level monitor request starting"); + log.debug("get rule field mappings request being made"); List ruleCategories = logIndexToQueries.getRight().stream().map(Pair::getRight).map(Rule::getCategory).distinct().collect( Collectors.toList()); Map queryBackendMap = new HashMap<>(); for(String category: ruleCategories){ - queryBackendMap.put(category, new OSQueryBackend(category, true, true)); + try { + queryBackendMap.put(category, new OSQueryBackend(category, true, true)); + } catch (IOException e) { + logger.error("Failed to create OSQueryBackend from category", e); + listener.onFailure(e); + } } List monitorRequests = new ArrayList<>(); + GroupedActionListener bucketLevelMonitorRequestsListener = new GroupedActionListener<>( + new ActionListener<>() { + @Override + public void onResponse(Collection indexMonitorRequests) { + listener.onResponse(monitorRequests); + } + + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, logIndexToQueries.getRight().size()); for (Pair query: logIndexToQueries.getRight()) { Rule rule = query.getRight(); // Creating bucket level monitor per each aggregation rule - if (rule.getAggregationQueries() != null){ - monitorRequests.add(createBucketLevelMonitorRequest( + if (rule.getAggregationQueries() != null) { + createBucketLevelMonitorRequest( query.getRight(), logIndexToQueries.getLeft(), detector, refreshPolicy, Monitor.NO_ID, Method.POST, - queryBackendMap.get(rule.getCategory()))); + queryBackendMap.get(rule.getCategory()), + new ActionListener<>() { + @Override + public void onResponse(IndexMonitorRequest indexMonitorRequest) { + monitorRequests.add(indexMonitorRequest); + bucketLevelMonitorRequestsListener.onResponse(indexMonitorRequest); + } + + @Override + public void onFailure(Exception e) { + logger.error("Failed to build bucket level monitor requests", e); + bucketLevelMonitorRequestsListener.onFailure(e); + } + }); + } else { + log.debug("Aggregation query is null in rule {}", rule.getId()); + bucketLevelMonitorRequestsListener.onResponse(null); } } - return monitorRequests; } - private IndexMonitorRequest createBucketLevelMonitorRequest( + private void createBucketLevelMonitorRequest( Rule rule, String index, Detector detector, WriteRequest.RefreshPolicy refreshPolicy, String monitorId, RestRequest.Method restMethod, - QueryBackend queryBackend - ) throws SigmaError { - AggregationQueries aggregationQueries = queryBackend.convertAggregation(rule.getAggregationItemsFromRule().get(0)); - - SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() - .seqNoAndPrimaryTerm(true) - .version(true) - // Build query string filter - .query(QueryBuilders.queryStringQuery(rule.getQueries().get(0).getValue())) - .aggregation(aggregationQueries.getAggBuilder()); - String concreteIndex = IndexUtils.getNewIndexByCreationDate( // index variable in method signature can also be an index pattern - clusterService.state(), - indexNameExpressionResolver, - index - ); + QueryBackend queryBackend, + ActionListener listener + ) { + log.debug(":create bucket level monitor response starting"); try { - GetIndexMappingsResponse getIndexMappingsResponse = client.execute( - GetIndexMappingsAction.INSTANCE, - new GetIndexMappingsRequest(concreteIndex)) - .actionGet(); - MappingMetadata mappingMetadata = getIndexMappingsResponse.mappings().get(concreteIndex); - List> pairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); - boolean timeStampAliasPresent = pairs. - stream() - .anyMatch(p -> - TIMESTAMP_FIELD_ALIAS.equals(p.getLeft()) || TIMESTAMP_FIELD_ALIAS.equals(p.getRight())); - if(timeStampAliasPresent) { - BoolQueryBuilder boolQueryBuilder = searchSourceBuilder.query() == null - ? new BoolQueryBuilder() - : QueryBuilders.boolQuery().must(searchSourceBuilder.query()); - RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS) - .gt("{{period_end}}||-1h") - .lte("{{period_end}}") - .format("epoch_millis"); - boolQueryBuilder.must(timeRangeFilter); - searchSourceBuilder.query(boolQueryBuilder); - } - } catch (Exception e) { - log.error( - String.format(Locale.getDefault(), - "Unable to verify presence of timestamp alias for index [%s] in detector [%s]. Not setting time range filter for bucket level monitor.", - concreteIndex, detector.getName()), e); - } - - List bucketLevelMonitorInputs = new ArrayList<>(); - bucketLevelMonitorInputs.add(new SearchInput(Arrays.asList(index), searchSourceBuilder)); - - List triggers = new ArrayList<>(); - BucketLevelTrigger bucketLevelTrigger = new BucketLevelTrigger(rule.getId(), rule.getTitle(), rule.getLevel(), aggregationQueries.getCondition(), - Collections.emptyList()); - triggers.add(bucketLevelTrigger); - - /** TODO - Think how to use detector trigger - List detectorTriggers = detector.getTriggers(); - for (DetectorTrigger detectorTrigger: detectorTriggers) { - String id = detectorTrigger.getId(); - String name = detectorTrigger.getName(); - String severity = detectorTrigger.getSeverity(); - List actions = detectorTrigger.getActions(); - Script condition = detectorTrigger.convertToCondition(); + AggregationQueries aggregationQueries = queryBackend.convertAggregation(rule.getAggregationItemsFromRule().get(0)); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder() + .seqNoAndPrimaryTerm(true) + .version(true) + // Build query string filter + .query(QueryBuilders.queryStringQuery(rule.getQueries().get(0).getValue())) + .aggregation(aggregationQueries.getAggBuilder()); + String concreteIndex = IndexUtils.getNewIndexByCreationDate( // index variable in method signature can also be an index pattern + clusterService.state(), + indexNameExpressionResolver, + index + ); - BucketLevelTrigger bucketLevelTrigger1 = new BucketLevelTrigger(id, name, severity, condition, actions); - triggers.add(bucketLevelTrigger1); - } **/ + client.execute( + GetIndexMappingsAction.INSTANCE, + new GetIndexMappingsRequest(concreteIndex), + new ActionListener<>() { + @Override + public void onResponse(GetIndexMappingsResponse getIndexMappingsResponse) { + MappingMetadata mappingMetadata = getIndexMappingsResponse.mappings().get(concreteIndex); + List> pairs = null; + try { + pairs = MapperUtils.getAllAliasPathPairs(mappingMetadata); + } catch (IOException e) { + logger.debug("Failed to get alias path pairs from mapping metadata", e); + onFailure(e); + } + boolean timeStampAliasPresent = pairs. + stream() + .anyMatch(p -> + TIMESTAMP_FIELD_ALIAS.equals(p.getLeft()) || TIMESTAMP_FIELD_ALIAS.equals(p.getRight())); + if(timeStampAliasPresent) { + BoolQueryBuilder boolQueryBuilder = searchSourceBuilder.query() == null + ? new BoolQueryBuilder() + : QueryBuilders.boolQuery().must(searchSourceBuilder.query()); + RangeQueryBuilder timeRangeFilter = QueryBuilders.rangeQuery(TIMESTAMP_FIELD_ALIAS) + .gt("{{period_end}}||-1h") + .lte("{{period_end}}") + .format("epoch_millis"); + boolQueryBuilder.must(timeRangeFilter); + searchSourceBuilder.query(boolQueryBuilder); + } - Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), detector.getEnabled(), detector.getSchedule(), detector.getLastUpdateTime(), detector.getEnabledTime(), - MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), - new DataSources(detector.getRuleIndex(), - detector.getFindingsIndex(), - detector.getFindingsIndexPattern(), - detector.getAlertsIndex(), - detector.getAlertsHistoryIndex(), - detector.getAlertsHistoryIndexPattern(), - DetectorMonitorConfig.getRuleIndexMappingsByType(detector.getDetectorType()), - true), PLUGIN_OWNER_FIELD); + List bucketLevelMonitorInputs = new ArrayList<>(); + bucketLevelMonitorInputs.add(new SearchInput(Arrays.asList(index), searchSourceBuilder)); + + List triggers = new ArrayList<>(); + BucketLevelTrigger bucketLevelTrigger = new BucketLevelTrigger(rule.getId(), rule.getTitle(), rule.getLevel(), aggregationQueries.getCondition(), + Collections.emptyList()); + triggers.add(bucketLevelTrigger); + + /** TODO - Think how to use detector trigger + List detectorTriggers = detector.getTriggers(); + for (DetectorTrigger detectorTrigger: detectorTriggers) { + String id = detectorTrigger.getId(); + String name = detectorTrigger.getName(); + String severity = detectorTrigger.getSeverity(); + List actions = detectorTrigger.getActions(); + Script condition = detectorTrigger.convertToCondition(); + + BucketLevelTrigger bucketLevelTrigger1 = new BucketLevelTrigger(id, name, severity, condition, actions); + triggers.add(bucketLevelTrigger1); + } **/ + + Monitor monitor = new Monitor(monitorId, Monitor.NO_VERSION, detector.getName(), detector.getEnabled(), detector.getSchedule(), detector.getLastUpdateTime(), detector.getEnabledTime(), + MonitorType.BUCKET_LEVEL_MONITOR, detector.getUser(), 1, bucketLevelMonitorInputs, triggers, Map.of(), + new DataSources(detector.getRuleIndex(), + detector.getFindingsIndex(), + detector.getFindingsIndexPattern(), + detector.getAlertsIndex(), + detector.getAlertsHistoryIndex(), + detector.getAlertsHistoryIndexPattern(), + DetectorMonitorConfig.getRuleIndexMappingsByType(detector.getDetectorType()), + true), PLUGIN_OWNER_FIELD); + + listener.onResponse(new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null)); + } - return new IndexMonitorRequest(monitorId, SequenceNumbers.UNASSIGNED_SEQ_NO, SequenceNumbers.UNASSIGNED_PRIMARY_TERM, refreshPolicy, restMethod, monitor, null); + @Override + public void onFailure(Exception e) { + log.error( + String.format(Locale.getDefault(), + "Unable to verify presence of timestamp alias for index [%s] in detector [%s]. Not setting time range filter for bucket level monitor.", + concreteIndex, detector.getName()), e); + listener.onFailure(e); + } + }); + } catch (SigmaError e) { + log.error("Failed to create bucket level monitor request", e); + listener.onFailure(e); + } } /** @@ -674,17 +819,22 @@ class AsyncIndexDetectorsAction { } void start() { + log.debug("stash context"); try { TransportIndexDetectorAction.this.threadPool.getThreadContext().stashContext(); + log.debug("log type check : {}", request.getDetector().getDetectorType()); if (!detectorIndices.detectorIndexExists()) { + log.debug("detector index creation"); detectorIndices.initDetectorIndex(new ActionListener<>() { @Override public void onResponse(CreateIndexResponse response) { + log.debug("detector index created in {}"); try { onCreateMappingsResponse(response); prepareDetectorIndexing(); } catch (IOException e) { + log.debug("detector index creation failed", e); onFailures(e); } } @@ -695,16 +845,19 @@ public void onFailure(Exception e) { } }); } else if (!IndexUtils.detectorIndexUpdated) { + log.debug("detector index update mapping"); IndexUtils.updateIndexMapping( Detector.DETECTORS_INDEX, DetectorIndices.detectorMappings(), clusterService.state(), client.admin().indices(), new ActionListener<>() { @Override public void onResponse(AcknowledgedResponse response) { + log.debug("detector index mapping updated"); onUpdateMappingsResponse(response); try { prepareDetectorIndexing(); } catch (IOException e) { + log.debug("detector index mapping FAILED updation", e); onFailures(e); } } @@ -749,24 +902,29 @@ void createDetector() { if (!detector.getInputs().isEmpty()) { try { + log.debug("init rule index template"); ruleTopicIndices.initRuleTopicIndexTemplate(new ActionListener<>() { @Override public void onResponse(AcknowledgedResponse acknowledgedResponse) { + log.debug("init rule index template ack"); initRuleIndexAndImportRules(request, new ActionListener<>() { @Override public void onResponse(List monitorResponses) { + log.debug("monitors indexed"); request.getDetector().setMonitorIds(getMonitorIds(monitorResponses)); request.getDetector().setRuleIdMonitorIdMap(mapMonitorIds(monitorResponses)); try { indexDetector(); - } catch (IOException e) { + } catch (Exception e) { + logger.debug("create detector failed", e); onFailures(e); } } @Override public void onFailure(Exception e) { + logger.debug("import rules failed", e); onFailures(e); } }); @@ -774,10 +932,12 @@ public void onFailure(Exception e) { @Override public void onFailure(Exception e) { + logger.debug("init rules index failed", e); onFailures(e); } }); - } catch (IOException e) { + } catch (Exception e) { + logger.debug("init rules index failed", e); onFailures(e); } } @@ -893,11 +1053,13 @@ public void initRuleIndexAndImportRules(IndexDetectorRequest request, ActionList new ActionListener<>() { @Override public void onResponse(CreateIndexResponse response) { + log.debug("prepackaged rule index created"); ruleIndices.onCreateMappingsResponse(response, true); ruleIndices.importRules(RefreshPolicy.IMMEDIATE, indexTimeout, new ActionListener<>() { @Override public void onResponse(BulkResponse response) { + log.debug("rules imported"); if (!response.hasFailures()) { importRules(request, listener); } else { @@ -907,6 +1069,7 @@ public void onResponse(BulkResponse response) { @Override public void onFailure(Exception e) { + log.debug("failed to import rules", e); onFailures(e); } }); @@ -1018,12 +1181,14 @@ public void importRules(IndexDetectorRequest request, ActionListener() { @Override public void onResponse(SearchResponse response) { if (response.isTimedOut()) { onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); } + logger.debug("prepackaged rules fetch success"); SearchHits hits = response.getHits(); List> queries = new ArrayList<>(); @@ -1046,13 +1211,10 @@ public void onResponse(SearchResponse response) { } else if (detectorInput.getCustomRules().size() > 0) { onFailures(new OpenSearchStatusException("Custom Rule Index not found", RestStatus.NOT_FOUND)); } else { - if (request.getMethod() == RestRequest.Method.POST) { - createMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } else if (request.getMethod() == RestRequest.Method.PUT) { - updateMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } + resolveRuleFieldNamesAndUpsertMonitorFromQueries(queries, detector, logIndex, listener); } - } catch (IOException | SigmaError e) { + } catch (Exception e) { + logger.debug("failed to fetch prepackaged rules", e); onFailures(e); } } @@ -1064,6 +1226,60 @@ public void onFailure(Exception e) { }); } + private void resolveRuleFieldNamesAndUpsertMonitorFromQueries(List> queries, Detector detector, String logIndex, ActionListener> listener) { + logger.error("PERF_DEBUG_SAP: Fetching alias path pairs to construct rule_field_names"); + long start = System.currentTimeMillis(); + Set ruleFieldNames = new HashSet<>(); + + for (Pair query : queries) { + List queryFieldNames = query.getValue().getQueryFieldNames().stream().map(Value::getValue).collect(Collectors.toList()); + ruleFieldNames.addAll(queryFieldNames); + } + client.execute(GetIndexMappingsAction.INSTANCE, new GetIndexMappingsRequest(logIndex), new ActionListener<>() { + @Override + public void onResponse(GetIndexMappingsResponse getMappingsViewResponse) { + try { + List> aliasPathPairs; + + aliasPathPairs = MapperUtils.getAllAliasPathPairs(getMappingsViewResponse.getMappings().get(logIndex)); + for (Pair aliasPathPair : aliasPathPairs) { + if (ruleFieldNames.contains(aliasPathPair.getLeft())) { + ruleFieldNames.remove(aliasPathPair.getLeft()); + ruleFieldNames.add(aliasPathPair.getRight()); + } + } + long took = System.currentTimeMillis() - start; + log.debug("completed collecting rule_field_names in {} millis", took); + } catch (Exception e) { + logger.error("Failure in parsing rule field names/aliases while " + + detector.getId() == null ? "creating" : "updating" + + " detector. Not optimizing detector queries with relevant fields", e); + ruleFieldNames.clear(); + } + try { + upsertMonitorQueries(queries, detector, listener, ruleFieldNames, logIndex); + } catch (Exception e) { + log.error("Caught exception upserting monitor queries", e); + listener.onFailure(e); + } + } + + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch mappings view response for log index " + logIndex, e); + listener.onFailure(e); + } + }); + } + + private void upsertMonitorQueries(List> queries, Detector detector, ActionListener> listener, Set ruleFieldNames, String logIndex) throws SigmaError, IOException { + if (request.getMethod() == Method.POST) { + createMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy(), new ArrayList<>(ruleFieldNames)); + } else if (request.getMethod() == Method.PUT) { + updateMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy(), new ArrayList<>(ruleFieldNames)); + } + } + @SuppressWarnings("unchecked") public void importCustomRules(Detector detector, DetectorInput detectorInput, List> queries, ActionListener> listener) { final String logIndex = detectorInput.getIndices().get(0); @@ -1077,6 +1293,7 @@ public void importCustomRules(Detector detector, DetectorInput detectorInput, Li .query(queryBuilder) .size(10000)); + logger.debug("importing custom rules"); client.search(searchRequest, new ActionListener<>() { @Override public void onResponse(SearchResponse response) { @@ -1084,6 +1301,7 @@ public void onResponse(SearchResponse response) { onFailures(new OpenSearchStatusException(response.toString(), RestStatus.REQUEST_TIMEOUT)); } + logger.debug("custom rules fetch successful"); SearchHits hits = response.getHits(); try { @@ -1099,12 +1317,8 @@ public void onResponse(SearchResponse response) { queries.add(Pair.of(id, rule)); } - if (request.getMethod() == RestRequest.Method.POST) { - createMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } else if (request.getMethod() == RestRequest.Method.PUT) { - updateMonitorFromQueries(logIndex, queries, detector, listener, request.getRefreshPolicy()); - } - } catch (IOException | SigmaError ex) { + resolveRuleFieldNamesAndUpsertMonitorFromQueries(queries, detector, logIndex, listener); + } catch (Exception ex) { onFailures(ex); } } @@ -1131,9 +1345,11 @@ public void indexDetector() throws IOException { .timeout(indexTimeout); } + log.debug("indexing detector"); client.index(indexRequest, new ActionListener<>() { @Override public void onResponse(IndexResponse response) { + log.debug("detector indexed success."); Detector responseDetector = request.getDetector(); responseDetector.setId(response.getId()); onOperation(response, responseDetector); From dbb52e00656d5f65481161d5b806d3a425f492d9 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 14 Mar 2024 18:34:58 -0700 Subject: [PATCH 07/11] Fix miss from manual cherry-pick Signed-off-by: Chase Engelbrecht --- .../TransportIndexDetectorAction.java | 52 ++++++++++++------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index e3a779d57..e34a8f617 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -269,23 +269,7 @@ private void createMonitorFromQueries(String index, List> rul if (numberOfUnprocessedResponses == 0) { listener.onResponse(monitorResponses); } else { - GroupedActionListener monitorResponseListener = new GroupedActionListener( - new ActionListener>() { - @Override - public void onResponse(Collection indexMonitorResponse) { - monitorResponses.addAll(indexMonitorResponse.stream().collect(Collectors.toList())); - listener.onResponse(monitorResponses); - } - - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }, numberOfUnprocessedResponses); - - for (int i = 1; i < monitorRequests.size(); i++) { - AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); - } + saveMonitors(monitorRequests, monitorResponses, numberOfUnprocessedResponses, listener); } }, listener::onFailure); }, listener::onFailure); @@ -300,13 +284,45 @@ public void onFailure(Exception e) { // Indexing monitors in two steps in order to prevent all shards failed error from alerting // https://github.com/opensearch-project/alerting/issues/646 AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(0), namedWriteableRegistry, indexDocLevelMonitorStep); - indexDocLevelMonitorStep.whenComplete(monitorResponses::add, listener::onFailure); + indexDocLevelMonitorStep.whenComplete(addedFirstMonitorResponse -> { + log.debug("first monitor created id {} of type {}", addedFirstMonitorResponse.getId(), addedFirstMonitorResponse.getMonitor().getMonitorType()); + monitorResponses.add(addedFirstMonitorResponse); + int numberOfUnprocessedResponses = monitorRequests.size() - 1; + if (numberOfUnprocessedResponses == 0) { + listener.onResponse(monitorResponses); + } else { + saveMonitors(monitorRequests, monitorResponses, numberOfUnprocessedResponses, listener); + } + }, listener::onFailure); } } catch (Exception ex) { listener.onFailure(ex); } } + private void saveMonitors( + List monitorRequests, + List monitorResponses, + int numberOfUnprocessedResponses, + ActionListener> listener + ) { + GroupedActionListener monitorResponseListener = new GroupedActionListener( + new ActionListener>() { + @Override + public void onResponse(Collection indexMonitorResponses) { + monitorResponses.addAll(indexMonitorResponses.stream().collect(Collectors.toList())); + listener.onResponse(monitorResponses); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }, numberOfUnprocessedResponses); + for (int i = 1; i < monitorRequests.size(); i++) { + AlertingPluginInterface.INSTANCE.indexMonitor((NodeClient) client, monitorRequests.get(i), namedWriteableRegistry, monitorResponseListener); + } + } + private void updateMonitorFromQueries(String index, List> rulesById, Detector detector, ActionListener> listener, WriteRequest.RefreshPolicy refreshPolicy, List queryFieldNames) throws IOException { List monitorsToBeUpdated = new ArrayList<>(); From 8b7660afb951f8c59ed28404b6504bf05ec3debb Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 14 Mar 2024 18:58:35 -0700 Subject: [PATCH 08/11] Undo exceptional case not originally present Signed-off-by: Chase Engelbrecht --- .../opensearch/securityanalytics/mapper/MapperService.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java index 882d056ba..f52e254fc 100644 --- a/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java +++ b/src/main/java/org/opensearch/securityanalytics/mapper/MapperService.java @@ -296,13 +296,6 @@ public void onResponse(GetMappingsResponse getMappingsResponse) { } } - if (appliedAliases.size() == 0) { - actionListener.onFailure(SecurityAnalyticsException.wrap( - new OpenSearchStatusException("No applied aliases found", RestStatus.NOT_FOUND)) - ); - return; - } - // Traverse mappings and do copy with excluded type=alias properties MappingsTraverser mappingsTraverser = new MappingsTraverser(mappingMetadata); // Resulting mapping after filtering From 053b703a9377a9a6f9200299cb10c2e7d186ef6c Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 14 Mar 2024 19:18:33 -0700 Subject: [PATCH 09/11] Revert test from previous commit reversion Signed-off-by: Chase Engelbrecht --- .../securityanalytics/mapper/MapperRestApiIT.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index a73887671..992166bb2 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -270,13 +270,13 @@ public void testUpdateAndGetMapping_notFound_Success() throws IOException { // Execute GetIndexMappingsAction and verify mappings Request getRequest = new Request("GET", SecurityAnalyticsPlugin.MAPPER_BASE_URI); getRequest.addParameter("index_name", testIndexName); - try { - client().performRequest(getRequest); - fail(); - } catch (ResponseException e) { - assertEquals(HttpStatus.SC_NOT_FOUND, e.getResponse().getStatusLine().getStatusCode()); - assertTrue(e.getMessage().contains("No applied aliases found")); - } + response = client().performRequest(getRequest); + XContentParser parser = createParser(JsonXContent.jsonXContent, new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8)); + assertTrue( + (((Map)((Map)parser.map() + .get(testIndexName)) + .get("mappings")) + .containsKey("properties"))); } public void testExistingMappingsAreUntouched() throws IOException { From 4ee977dcebc8a63abcf37da8cf9098e798d337a0 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 14 Mar 2024 19:47:49 -0700 Subject: [PATCH 10/11] Fix another miss from manual cherry-pick Signed-off-by: Chase Engelbrecht --- .../transport/TransportIndexDetectorAction.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index e34a8f617..c21f7d3f8 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -415,6 +415,17 @@ public void onFailure(Exception e) { } } } + } else { + onIndexMonitorRequestCreation( + index, + monitorsToBeUpdated, + monitorsToBeAdded, + rulesById, + detector, + refreshPolicy, + queryFieldNames, + listener + ); } } From ef94468a8b21fb95e4fecabbfd953df433c670f0 Mon Sep 17 00:00:00 2001 From: Chase Engelbrecht Date: Thu, 14 Mar 2024 20:00:19 -0700 Subject: [PATCH 11/11] Revert behavior for no rules in detector Signed-off-by: Chase Engelbrecht --- .../transport/TransportIndexDetectorAction.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index c21f7d3f8..cf9835568 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -274,9 +274,9 @@ private void createMonitorFromQueries(String index, List> rul }, listener::onFailure); }, listener::onFailure); } else { - // Failure if detector doesn't have any monitor + // Do nothing if detector doesn't have any monitor if (monitorRequests.isEmpty()) { - listener.onFailure(new OpenSearchStatusException("Detector cannot be created as no compatible rules were provided", RestStatus.BAD_REQUEST)); + listener.onResponse(Collections.emptyList()); return; } List monitorResponses = new ArrayList<>();