From 82e73ab9b1c076651ae5405140424c3f88a1fcf6 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Tue, 1 Aug 2017 17:59:28 -0500 Subject: [PATCH 1/2] initial commit --- .../elasticsearch/dao/ElasticsearchDao.java | 20 ++++++++- .../indexing/dao/search/SearchRequest.java | 10 +++++ .../dao/IndexingDaoIntegrationTest.java | 42 +++++++++++++++++-- 3 files changed, 67 insertions(+), 5 deletions(-) diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index cb2b1ca7be..ecdb09b4d9 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -37,6 +37,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.stream.Collectors; public class ElasticsearchDao implements IndexDao { @@ -81,8 +82,14 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx .size(searchRequest.getSize()) .from(searchRequest.getFrom()) .query(new QueryStringQueryBuilder(searchRequest.getQuery())) - .fetchSource(true) + .trackScores(true); + Optional> fields = searchRequest.getFields(); + if (fields.isPresent()) { + searchSourceBuilder.fields(fields.get()); + } else { + searchSourceBuilder.fetchSource(true); + } for (SortField sortField : searchRequest.getSort()) { FieldSortBuilder fieldSortBuilder = new FieldSortBuilder(sortField.getField()); if (sortField.getSortOrder() == org.apache.metron.indexing.dao.search.SortOrder.DESC) { @@ -100,7 +107,16 @@ public SearchResponse search(SearchRequest searchRequest) throws InvalidSearchEx searchResponse.setResults(Arrays.stream(elasticsearchResponse.getHits().getHits()).map(searchHit -> { SearchResult searchResult = new SearchResult(); searchResult.setId(searchHit.getId()); - searchResult.setSource(searchHit.getSource()); + Map source; + if (fields.isPresent()) { + source = new HashMap<>(); + searchHit.getFields().forEach((key, value) -> { + source.put(key, value.getValues().size() == 1 ? value.getValue() : value.getValues()); + }); + } else { + source = searchHit.getSource(); + } + searchResult.setSource(source); searchResult.setScore(searchHit.getScore()); return searchResult; }).collect(Collectors.toList())); diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java index ecf6b57f4f..ce38278083 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/search/SearchRequest.java @@ -19,6 +19,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; public class SearchRequest { @@ -27,6 +28,7 @@ public class SearchRequest { private int size; private int from; private List sort; + private List fields; public SearchRequest() { SortField defaultSortField = new SortField(); @@ -75,4 +77,12 @@ public List getSort() { public void setSort(List sort) { this.sort = sort; } + + public Optional> getFields() { + return fields == null || fields.size() == 0 ? Optional.empty() : Optional.of(fields); + } + + public void setFields(List fields) { + this.fields = fields; + } } diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java index 209c2349c2..fc0b370443 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/IndexingDaoIntegrationTest.java @@ -25,10 +25,11 @@ import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.integration.InMemoryComponent; -import org.json.simple.parser.ParseException; -import org.junit.*; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; -import java.io.IOException; import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -161,6 +162,24 @@ public abstract class IndexingDaoIntegrationTest { @Multiline public static String exceededMaxResultsQuery; + /** + * { + * "fields": ["ip_src_addr"], + * "indices": ["bro", "snort"], + * "query": "*", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String fieldsQuery; + protected IndexDao dao; protected InMemoryComponent indexComponent; @@ -323,6 +342,23 @@ public void test() throws Exception { Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field")); } + //Fields query + { + SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(10, response.getTotal()); + List results = response.getResults(); + for(int i = 0;i < 5;++i) { + Map source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + for(int i = 5;i < 10;++i) { + Map source = results.get(i).getSource(); + Assert.assertEquals(1, source.size()); + Assert.assertNotNull(source.get("ip_src_addr")); + } + } } @After From 064ec501156479108f3817cc9108905bacd29f46 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Wed, 23 Aug 2017 15:22:15 -0500 Subject: [PATCH 2/2] added test case for empty result set --- .../indexing/dao/SearchIntegrationTest.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 5750ce73a1..2645df2f22 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -235,8 +235,24 @@ public abstract class SearchIntegrationTest { @Multiline public static String fieldsQuery; - protected IndexDao dao; - protected InMemoryComponent indexComponent; + /** + * { + * "fields": ["ip_src_addr"], + * "indices": ["bro", "snort"], + * "query": "ip_src_addr:192.168.1.9", + * "from": 0, + * "size": 10, + * "sort": [ + * { + * "field": "timestamp", + * "sortOrder": "desc" + * } + * ] + * } + */ + @Multiline + public static String noResultsFieldsQuery; + protected static IndexDao dao; protected static InMemoryComponent indexComponent; @@ -505,6 +521,12 @@ public void test() throws Exception { Assert.assertNotNull(source.get("ip_src_addr")); } } + //No results fields query + { + SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class); + SearchResponse response = dao.search(request); + Assert.assertEquals(0, response.getTotal()); + } } @AfterClass