From 6c9317bddfee6cb98121d2443bfcc28d8852a2f0 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 22 Sep 2017 11:33:24 -0400 Subject: [PATCH 1/2] Killing sleep call --- .../org/apache/metron/elasticsearch/dao/ElasticsearchDao.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index 0a06c806f8..62706e3a09 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -308,9 +308,6 @@ public void update(Document update, Optional index) throws IOException { if (failed > 0) { throw new IOException("ElasticsearchDao upsert failed: " + Arrays.toString(shardInfo.getFailures())); } - Thread.sleep(10000); - org.elasticsearch.action.search.SearchResponse resultAfter = client.prepareSearch("test*").setFetchSource(true).setQuery(QueryBuilders.matchAllQuery()).get(); - resultAfter.getHits(); } catch (Exception e) { throw new IOException(e.getMessage(), e); } From f6c63c9109d33d8bf014c205edab20daf8b49949 Mon Sep 17 00:00:00 2001 From: justinjleet Date: Fri, 22 Sep 2017 12:17:10 -0400 Subject: [PATCH 2/2] Adding retries to initial grab from ES --- ...ElasticsearchMetaAlertIntegrationTest.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index fda62ab31c..c562e0ba76 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -198,8 +198,8 @@ public void test() throws Exception { { //ensure alerts in ES are up-to-date - Document doc = metaDao.getLatest(guid, SENSOR_NAME); - Assert.assertEquals(message0, doc.getDocument()); + boolean found = findUpdatedDoc(message0, guid); + Assert.assertTrue("Unable to find updated document", found); long cnt = 0; for (int t = 0; t < MAX_RETRIES && cnt == 0; ++t, Thread.sleep(SLEEP_MS)) { docs = es.getAllIndexedDocs(INDEX, SENSOR_NAME + "_doc"); @@ -257,8 +257,8 @@ public void test() throws Exception { } }, Optional.empty()); - Document doc = metaDao.getLatest(guid, SENSOR_NAME); - Assert.assertEquals(message0, doc.getDocument()); + boolean found = findUpdatedDoc(message0, guid); + Assert.assertTrue("Unable to find updated document", found); { //ensure ES is up-to-date long cnt = 0; @@ -302,6 +302,18 @@ public void test() throws Exception { } } + protected boolean findUpdatedDoc(Map message0, String guid) + throws InterruptedException, IOException { + boolean found = false; + for (int t = 0; t < MAX_RETRIES && !found; ++t, Thread.sleep(SLEEP_MS)) { + Document doc = metaDao.getLatest(guid, SENSOR_NAME); + if (message0.equals(doc.getDocument())) { + found = true; + } + } + return found; + } + protected void elasticsearchAdd(List> inputData, String index, String docType) throws IOException { es.add(index, docType, inputData.stream().map(m -> {