From 28bb5c13199a483d9c8c51bb782293050a429fe9 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Tue, 17 Oct 2017 11:15:24 -0500 Subject: [PATCH 1/3] initial commit --- .../dao/ElasticsearchMetaAlertDao.java | 12 +- ...ElasticsearchMetaAlertIntegrationTest.java | 132 +++++++++++++++++- 2 files changed, 135 insertions(+), 9 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index a64f5ad5b6..b71d156eba 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -21,6 +21,7 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; import static org.elasticsearch.index.query.QueryBuilders.termQuery; @@ -177,15 +178,18 @@ public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchException { // Wrap the query to also get any meta-alerts. QueryBuilder qb = constantScoreQuery(boolQuery() - .should(new QueryStringQueryBuilder(searchRequest.getQuery())) - .should(boolQuery() - .must(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString())) - .must(nestedQuery( + .must(boolQuery() + .should(new QueryStringQueryBuilder(searchRequest.getQuery())) + .should(nestedQuery( ALERT_FIELD, new QueryStringQueryBuilder(searchRequest.getQuery()) ) ) ) + .must(boolQuery() + .should(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString())) + .should(boolQuery().mustNot(existsQuery(MetaAlertDao.STATUS_FIELD))) + ) ); return elasticsearchDao.search(searchRequest, qb); } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index c562e0ba76..5c85df62ba 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -21,16 +21,20 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import java.io.File; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.common.Constants; import org.apache.metron.common.utils.JSONUtils; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; @@ -40,6 +44,9 @@ import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MetaAlertDao; +import org.apache.metron.indexing.dao.search.SearchRequest; +import org.apache.metron.indexing.dao.search.SearchResponse; +import org.apache.metron.indexing.dao.search.SortField; import org.apache.metron.indexing.dao.update.Document; import org.apache.metron.indexing.dao.update.ReplaceRequest; import org.junit.AfterClass; @@ -83,6 +90,7 @@ public static void setup() throws Exception { put("es.date.format", DATE_FORMAT); } }; + accessConfig.setMaxSearchResults(1000); accessConfig.setGlobalConfigSupplier(() -> globalConfig); esDao = new ElasticsearchDao(); @@ -198,7 +206,7 @@ public void test() throws Exception { { //ensure alerts in ES are up-to-date - boolean found = findUpdatedDoc(message0, guid); + boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME); Assert.assertTrue("Unable to find updated document", found); long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { @@ -257,7 +265,7 @@ public void test() throws Exception { } }, Optional.empty()); - boolean found = findUpdatedDoc(message0, guid); + boolean found = findUpdatedDoc(message0, guid, SENSOR_NAME); Assert.assertTrue("Unable to find updated document", found); { //ensure ES is up-to-date @@ -302,12 +310,126 @@ public void test() throws Exception { } } - protected boolean findUpdatedDoc(Map message0, String guid) + /** + { + "guid": "search_by_status_active", + "source:type": "metaalert", + "alert": [], + "status": "active" + } + */ + @Multiline + public static String activeMetaAlert; + + /** + { + "guid": "search_by_status_inactive", + "source:type": "metaalert", + "alert": [], + "status": "inactive" + } + */ + @Multiline + public static String inactiveMetaAlert; + + @Test + public void shouldSearchByStatus() throws Exception { + List> metaInputData = new ArrayList<>(); + Map activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert, new TypeReference>() {}); + metaInputData.add(activeMetaAlertJSON); + Map inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert, new TypeReference>() {}); + metaInputData.add(inactiveMetaAlertJSON); + + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + elasticsearchAdd(metaInputData, MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); + // Wait for updates to persist + findUpdatedDoc(inactiveMetaAlertJSON, "search_by_status_inactive", MetaAlertDao.METAALERT_TYPE); + + SearchResponse searchResponse = metaDao.search(new SearchRequest() { + { + setQuery("*"); + setIndices(Collections.singletonList(MetaAlertDao.METAALERTS_INDEX)); + setFrom(0); + setSize(5); + setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); + } + }); + Assert.assertEquals(1, searchResponse.getTotal()); + Assert.assertEquals(MetaAlertStatus.ACTIVE.getStatusString(), searchResponse.getResults().get(0).getSource().get(MetaAlertDao.STATUS_FIELD)); + } + + /** + { + "guid": "search_by_nested_alert_0", + "source:type": "test", + "ip_src_addr": "192.168.1.1", + "ip_src_port": 8010 + } + */ + @Multiline + public static String searchByNestedAlert0; + + /** + { + "guid": "search_by_nested_alert_1", + "source:type": "test", + "ip_src_addr": "192.168.1.2", + "ip_src_port": 8009 + } + */ + @Multiline + public static String searchByNestedAlert1; + + @Test + public void shouldSearchByNestedAlert() throws Exception { + List> inputData = new ArrayList<>(); + Map searchByNestedAlert0JSON = JSONUtils.INSTANCE.load(searchByNestedAlert0, new TypeReference>() {}); + inputData.add(searchByNestedAlert0JSON); + Map searchByNestedAlert1JSON = JSONUtils.INSTANCE.load(searchByNestedAlert1, new TypeReference>() {}); + inputData.add(searchByNestedAlert1JSON); + elasticsearchAdd(inputData, INDEX, SENSOR_NAME); + // Wait for updates to persist + findUpdatedDoc(searchByNestedAlert0JSON, "search_by_nested_alert_0", SENSOR_NAME); + + Map activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert, new TypeReference>() {}); + activeMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlert0JSON, searchByNestedAlert1JSON)); + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. + elasticsearchAdd(Collections.singletonList(activeMetaAlertJSON), MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); + // Wait for updates to persist + findUpdatedDoc(activeMetaAlertJSON, "search_by_status_active", MetaAlertDao.METAALERT_TYPE); + + SearchResponse searchResponse = metaDao.search(new SearchRequest() { + { + setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)"); + setIndices(Collections.singletonList(MetaAlertDao.METAALERTS_INDEX)); + setFrom(0); + setSize(5); + setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); + } + }); + // Should not have results because nested alerts shouldn't be flattened + Assert.assertEquals(0, searchResponse.getTotal()); + + searchResponse = metaDao.search(new SearchRequest() { + { + setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)"); + setIndices(Collections.singletonList(MetaAlertDao.METAALERTS_INDEX)); + setFrom(0); + setSize(5); + setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); + } + }); + // Nested query should match a nested alert + Assert.assertEquals(1, searchResponse.getTotal()); + Assert.assertEquals("search_by_status_active", searchResponse.getResults().get(0).getSource().get("guid")); + } + + protected boolean findUpdatedDoc(Map message0, String guid, String sensorType) throws InterruptedException, IOException { boolean found = false; for (int t = 0; t < MAX_RETRIES && !found; ++t, Thread.sleep(SLEEP_MS)) { - Document doc = metaDao.getLatest(guid, SENSOR_NAME); - if (message0.equals(doc.getDocument())) { + Document doc = metaDao.getLatest(guid, sensorType); + if (doc != null && message0.equals(doc.getDocument())) { found = true; } } From bc923540e3a33225d458db9505045cf062d553f2 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Tue, 17 Oct 2017 13:02:49 -0500 Subject: [PATCH 2/3] resolved merge conflicts --- .../integration/ElasticsearchMetaAlertIntegrationTest.java | 6 +++--- .../java/org/apache/metron/indexing/dao/MetaAlertDao.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 5c85df62ba..7256fad9d9 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -348,7 +348,7 @@ public void shouldSearchByStatus() throws Exception { SearchResponse searchResponse = metaDao.search(new SearchRequest() { { setQuery("*"); - setIndices(Collections.singletonList(MetaAlertDao.METAALERTS_INDEX)); + setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE)); setFrom(0); setSize(5); setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); @@ -401,7 +401,7 @@ public void shouldSearchByNestedAlert() throws Exception { SearchResponse searchResponse = metaDao.search(new SearchRequest() { { setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8009) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8009)"); - setIndices(Collections.singletonList(MetaAlertDao.METAALERTS_INDEX)); + setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE)); setFrom(0); setSize(5); setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); @@ -413,7 +413,7 @@ public void shouldSearchByNestedAlert() throws Exception { searchResponse = metaDao.search(new SearchRequest() { { setQuery("(ip_src_addr:192.168.1.1 AND ip_src_port:8010) OR (alert.ip_src_addr:192.168.1.1 AND alert.ip_src_port:8010)"); - setIndices(Collections.singletonList(MetaAlertDao.METAALERTS_INDEX)); + setIndices(Collections.singletonList(MetaAlertDao.METAALERT_TYPE)); setFrom(0); setSize(5); setSort(Collections.singletonList(new SortField(){{ setField(Constants.GUID); }})); diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java index 05746c49aa..e9f047b77a 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java @@ -27,7 +27,7 @@ public interface MetaAlertDao extends IndexDao { - String METAALERTS_INDEX = "metaalert"; + String METAALERTS_INDEX = "metaalert_index"; String METAALERT_TYPE = "metaalert"; String METAALERT_DOC = METAALERT_TYPE + "_doc"; String THREAT_FIELD_DEFAULT = "threat:triage:score"; From 57aba446aea87706b72128290ad49aae05072d9c Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 18 Oct 2017 15:27:48 -0500 Subject: [PATCH 3/3] addressed feedback --- .../dao/ElasticsearchMetaAlertDao.java | 1 + ...ElasticsearchMetaAlertIntegrationTest.java | 67 ++++++++++++++----- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index b71d156eba..44c983e7b4 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -186,6 +186,7 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx ) ) ) + // Ensures that it's a meta alert with active status or that it's an alert (signified by having no status field) .must(boolQuery() .should(termQuery(MetaAlertDao.STATUS_FIELD, MetaAlertStatus.ACTIVE.getStatusString())) .should(boolQuery().mustNot(existsQuery(MetaAlertDao.STATUS_FIELD))) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 7256fad9d9..12ac8884eb 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -312,7 +312,7 @@ public void test() throws Exception { /** { - "guid": "search_by_status_active", + "guid": "active_metaalert", "source:type": "metaalert", "alert": [], "status": "active" @@ -323,7 +323,7 @@ public void test() throws Exception { /** { - "guid": "search_by_status_inactive", + "guid": "inactive_metaalert", "source:type": "metaalert", "alert": [], "status": "inactive" @@ -343,7 +343,7 @@ public void shouldSearchByStatus() throws Exception { // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. elasticsearchAdd(metaInputData, MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); // Wait for updates to persist - findUpdatedDoc(inactiveMetaAlertJSON, "search_by_status_inactive", MetaAlertDao.METAALERT_TYPE); + findUpdatedDoc(inactiveMetaAlertJSON, "inactive_metaalert", MetaAlertDao.METAALERT_TYPE); SearchResponse searchResponse = metaDao.search(new SearchRequest() { { @@ -360,43 +360,74 @@ public void shouldSearchByStatus() throws Exception { /** { - "guid": "search_by_nested_alert_0", + "guid": "search_by_nested_alert_active_0", "source:type": "test", "ip_src_addr": "192.168.1.1", "ip_src_port": 8010 } */ @Multiline - public static String searchByNestedAlert0; + public static String searchByNestedAlertActive0; /** { - "guid": "search_by_nested_alert_1", + "guid": "search_by_nested_alert_inactive_1", "source:type": "test", "ip_src_addr": "192.168.1.2", "ip_src_port": 8009 } */ @Multiline - public static String searchByNestedAlert1; + public static String searchByNestedAlertActive1; + + /** + { + "guid": "search_by_nested_alert_inactive_0", + "source:type": "test", + "ip_src_addr": "192.168.1.3", + "ip_src_port": 8008 + } + */ + @Multiline + public static String searchByNestedAlertInactive0; + + /** + { + "guid": "search_by_nested_alert_inactive_1", + "source:type": "test", + "ip_src_addr": "192.168.1.4", + "ip_src_port": 8007 + } + */ + @Multiline + public static String searchByNestedAlertInactive1; @Test public void shouldSearchByNestedAlert() throws Exception { - List> inputData = new ArrayList<>(); - Map searchByNestedAlert0JSON = JSONUtils.INSTANCE.load(searchByNestedAlert0, new TypeReference>() {}); - inputData.add(searchByNestedAlert0JSON); - Map searchByNestedAlert1JSON = JSONUtils.INSTANCE.load(searchByNestedAlert1, new TypeReference>() {}); - inputData.add(searchByNestedAlert1JSON); - elasticsearchAdd(inputData, INDEX, SENSOR_NAME); + // Create alerts + List> alerts = new ArrayList<>(); + Map searchByNestedAlertActive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive0, new TypeReference>() {}); + alerts.add(searchByNestedAlertActive0JSON); + Map searchByNestedAlertActive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertActive1, new TypeReference>() {}); + alerts.add(searchByNestedAlertActive1JSON); + Map searchByNestedAlertInactive0JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive0, new TypeReference>() {}); + alerts.add(searchByNestedAlertInactive0JSON); + Map searchByNestedAlertInactive1JSON = JSONUtils.INSTANCE.load(searchByNestedAlertInactive1, new TypeReference>() {}); + alerts.add(searchByNestedAlertInactive1JSON); + elasticsearchAdd(alerts, INDEX, SENSOR_NAME); // Wait for updates to persist - findUpdatedDoc(searchByNestedAlert0JSON, "search_by_nested_alert_0", SENSOR_NAME); + findUpdatedDoc(searchByNestedAlertInactive1JSON, "search_by_nested_alert_inactive_1", SENSOR_NAME); + // Create metaalerts Map activeMetaAlertJSON = JSONUtils.INSTANCE.load(activeMetaAlert, new TypeReference>() {}); - activeMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlert0JSON, searchByNestedAlert1JSON)); + activeMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertActive0JSON, searchByNestedAlertActive1JSON)); + Map inactiveMetaAlertJSON = JSONUtils.INSTANCE.load(inactiveMetaAlert, new TypeReference>() {}); + inactiveMetaAlertJSON.put("alert", Arrays.asList(searchByNestedAlertInactive0JSON, searchByNestedAlertInactive1JSON)); + // We pass MetaAlertDao.METAALERT_TYPE, because the "_doc" gets appended automatically. - elasticsearchAdd(Collections.singletonList(activeMetaAlertJSON), MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); + elasticsearchAdd(Arrays.asList(activeMetaAlertJSON, inactiveMetaAlertJSON), MetaAlertDao.METAALERTS_INDEX, MetaAlertDao.METAALERT_TYPE); // Wait for updates to persist - findUpdatedDoc(activeMetaAlertJSON, "search_by_status_active", MetaAlertDao.METAALERT_TYPE); + findUpdatedDoc(activeMetaAlertJSON, "active_metaalert", MetaAlertDao.METAALERT_TYPE); SearchResponse searchResponse = metaDao.search(new SearchRequest() { { @@ -421,7 +452,7 @@ public void shouldSearchByNestedAlert() throws Exception { }); // Nested query should match a nested alert Assert.assertEquals(1, searchResponse.getTotal()); - Assert.assertEquals("search_by_status_active", searchResponse.getResults().get(0).getSource().get("guid")); + Assert.assertEquals("active_metaalert", searchResponse.getResults().get(0).getSource().get("guid")); } protected boolean findUpdatedDoc(Map message0, String guid, String sensorType)