Skip to content
This repository has been archived by the owner on May 12, 2021. It is now read-only.

METRON-1429: SearchIntegrationTest refactor #909

Closed
wants to merge 5 commits into from
Closed

METRON-1429: SearchIntegrationTest refactor #909

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7645598
initial commit
merrimanr Jan 24, 2018
a29ed14
added back metaalert test data
merrimanr Jan 24, 2018
5c6293d
Merge remote-tracking branch 'mirror/master' into METRON-1429
merrimanr Jan 24, 2018
257c019
added loops
merrimanr Jan 25, 2018
86c82b2
Merge remote-tracking branch 'mirror/master' into METRON-1429
merrimanr Jan 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
11 changes: 10 additions & 1 deletion ...t/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
View file
Open in desktop
Expand Up @@ -96,13 +96,22 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
@Multiline
public static String create;

/**
* [
*{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
*{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
* ]
*/
@Multiline
public static String metaAlertData;

@Before
public void setup() throws Exception {
this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
ImmutableMap<String, String> testData = ImmutableMap.of(
"bro_index_2017.01.01.01", SearchIntegrationTest.broData,
"snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
MetaAlertDao.METAALERTS_INDEX, metaAlertData
);
loadTestData(testData);
}
Expand Down
2 changes: 1 addition & 1 deletion ...rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
View file
Open in desktop
Expand Up @@ -121,7 +121,7 @@ public void setup() throws Exception {
ImmutableMap<String, String> testData = ImmutableMap.of(
"bro_index_2017.01.01.01", SearchIntegrationTest.broData,
"snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
MetaAlertDao.METAALERTS_INDEX, MetaAlertControllerIntegrationTest.metaAlertData
);
loadTestData(testData);
}
Expand Down
99 changes: 3 additions & 96 deletions .../java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
View file
Open in desktop
Expand Up @@ -19,35 +19,25 @@


import java.io.File;
import java.util.HashMap;
import java.io.IOException;
import java.util.HashMap;
import java.util.concurrent.ExecutionException;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.MetaAlertDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.apache.metron.integration.InMemoryComponent;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.SearchHit;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;

import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.concurrent.ExecutionException;

public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {

private static String indexDir = "target/elasticsearch_search";
Expand Down Expand Up @@ -181,34 +171,6 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
@Multiline
private static String broDefaultStringMappings;

/**
* {
* "metaalert_doc": {
* "properties": {
* "guid": { "type": "keyword" },
* "alert": {
* "type": "nested",
* "properties": {
* "guid": { "type": "keyword" }
* }
* },
* "average": { "type": "keyword" },
* "min" : { "type": "keyword" },
* "median" : { "type": "keyword" },
* "max": { "type": "keyword" },
* "count": { "type": "keyword" },
* "sum": { "type": "keyword" },
* "source:type": {
* "type": "text",
* "fielddata" : "true"
* }
* }
* }
* }
*/
@Multiline
private static String metaAlertTypeMappings;

@Override
protected IndexDao createDao() throws Exception {
AccessConfig config = new AccessConfig();
Expand Down Expand Up @@ -246,14 +208,13 @@ protected void loadTestData()
.addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get();
es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
.addMapping("snort_doc", snortTypeMappings).get();
es.getClient().admin().indices().prepareCreate(MetaAlertDao.METAALERTS_INDEX)
.addMapping(MetaAlertDao.METAALERT_DOC, metaAlertTypeMappings).get();

BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
for(Object o: broArray) {
JSONObject jsonObject = (JSONObject) o;
IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("bro_index_2017.01.01.01", "bro_doc");
indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
bulkRequest.add(indexRequestBuilder);
Expand All @@ -262,68 +223,14 @@ protected void loadTestData()
for(Object o: snortArray) {
JSONObject jsonObject = (JSONObject) o;
IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("snort_index_2017.01.01.02", "snort_doc");
indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
bulkRequest.add(indexRequestBuilder);
}
JSONArray metaAlertArray = (JSONArray) new JSONParser().parse(metaAlertData);
for(Object o: metaAlertArray) {
JSONObject jsonObject = (JSONObject) o;
IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalert_index", "metaalert_doc");
indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
bulkRequest.add(indexRequestBuilder);
}
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
throw new RuntimeException("Failed to index test data");
}

SearchResponse broDocs = es.getClient()
.prepareSearch("bro_index_2017.01.01.01")
.setTypes("bro_doc")
.setQuery(QueryBuilders.matchAllQuery())
.get();
// We're changing the _id field, we need to create a copy and delete the original.
for (SearchHit hit : broDocs.getHits()) {
// Bro GUIDs to collide while using the standard analyzer
// Use timestamp as part of guid because query may not return in order each time
IndexRequest indexRequest = new IndexRequest()
.index("bro_index_2017.01.01.01")
.type("bro_doc")
.id("bro-" + hit.getSource().get("timestamp"))
.source(hit.getSource());
es.getClient().index(indexRequest).get();

// Delete the original
es.getClient()
.prepareDelete("bro_index_2017.01.01.01", "bro_doc", hit.getId())
.get();
}

// Wait until everything is updated
// Assume true until proven otherwise.
boolean allUpdated = true;
for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
allUpdated = true;
SearchResponse response = es.getClient()
.prepareSearch("bro_index_2017.01.01.01")
.setTypes("bro_doc")
.setQuery(QueryBuilders.matchAllQuery())
.get();
if (response.getHits().getTotalHits() == 0) {
throw new IllegalStateException("Bro index is empty. No docs to validate were updated");
}
for (SearchHit hit : response.getHits()) {
if (!hit.getId().startsWith("bro-")) {
allUpdated = false;
}
}
if (allUpdated) {
break;
}
}
if (!allUpdated) {
throw new IllegalStateException("Unable to update Elasticsearch ids properly");
}
}
}