From 9c7576f7fe384e73540b5e561378d4f49a90694e Mon Sep 17 00:00:00 2001 From: merrimanr Date: Thu, 7 Jun 2018 09:24:37 -0500 Subject: [PATCH 1/2] initial commit --- .../rest/service/impl/SearchServiceImpl.java | 3 +- .../org/apache/metron/common/Constants.java | 1 + .../elasticsearch/dao/ElasticsearchDao.java | 8 +++ .../dao/ElasticsearchMetaAlertDao.java | 25 +++++--- .../dao/ElasticsearchMetaAlertDaoTest.java | 33 +++++++++- ...ElasticsearchMetaAlertIntegrationTest.java | 64 ++++++++++++++++--- .../metron/indexing/dao/MetaAlertDao.java | 3 +- 7 files changed, 115 insertions(+), 22 deletions(-) diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java index 21d158fa20..a001390ce2 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@ -18,6 +18,7 @@ package org.apache.metron.rest.service.impl; import static org.apache.metron.common.Constants.ERROR_TYPE; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY; @@ -151,7 +152,7 @@ public List getDefaultFacetFields() throws RestException { String sourceTypeField = Constants.SENSOR_TYPE.replace('.', ':'); List facetFields = new ArrayList<>(); if (globalConfig != null) { - sourceTypeField = (String) globalConfig.getOrDefault("source.type.field", sourceTypeField); + sourceTypeField = (String) globalConfig.getOrDefault(SENSOR_TYPE_FIELD_PROPERTY, sourceTypeField); } facetFields.add(sourceTypeField); if (facetFieldsProperty != null) { diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java index 12b541cd77..f74660c48f 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/Constants.java @@ -27,6 +27,7 @@ public class Constants { public static final String ZOOKEEPER_TOPOLOGY_ROOT = ZOOKEEPER_ROOT + "/topology"; public static final long DEFAULT_CONFIGURED_BOLT_TIMEOUT = 5000; public static final String SENSOR_TYPE = "source.type"; + public static final String SENSOR_TYPE_FIELD_PROPERTY = "source.type.field"; public static final String ENRICHMENT_TOPIC = "enrichments"; public static final String INDEXING_TOPIC = "indexing"; public static final String ERROR_STREAM = "error"; diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java index cb5bb58aa3..181cb8721b 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchDao.java @@ -125,6 +125,14 @@ public ElasticsearchDao() { //uninitialized. } + public AccessConfig getAccessConfig() { + return accessConfig; + } + + public void setAccessConfig(AccessConfig accessConfig) { + this.accessConfig = accessConfig; + } + private static Map elasticsearchSearchTypeMap; static { diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index d12e40ca5d..c3bb4a6f9d 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -19,6 +19,7 @@ package org.apache.metron.elasticsearch.dao; import static org.apache.metron.common.Constants.GUID; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; import static org.elasticsearch.index.query.QueryBuilders.existsQuery; @@ -78,13 +79,13 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { public static final String SOURCE_TYPE = Constants.SENSOR_TYPE.replace('.', ':'); + public static final String THREAT_TRIAGE_FIELD = THREAT_FIELD_DEFAULT.replace('.', ':'); private static final String STATUS_PATH = "/status"; private static final String ALERT_PATH = "/alert"; private IndexDao indexDao; private ElasticsearchDao elasticsearchDao; private String index = METAALERTS_INDEX; - private String threatTriageField = THREAT_FIELD_DEFAULT; /** * Defines which summary aggregation is used to represent the overall threat triage score for @@ -101,21 +102,19 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao { * @param indexDao The Dao to wrap */ public ElasticsearchMetaAlertDao(IndexDao indexDao) { - this(indexDao, METAALERTS_INDEX, THREAT_FIELD_DEFAULT, THREAT_SORT_DEFAULT); + this(indexDao, METAALERTS_INDEX, THREAT_SORT_DEFAULT); } /** * Wraps an {@link org.apache.metron.indexing.dao.IndexDao} to handle meta alerts. * @param indexDao The Dao to wrap - * @param triageLevelField The field name to use as the threat scoring field * @param threatSort The summary aggregation of all child threat triage scores used * as the overall threat triage score for the metaalert. This * can be either max, min, average, count, median, or sum. */ - public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String triageLevelField, String threatSort) { + public ElasticsearchMetaAlertDao(IndexDao indexDao, String index, String threatSort) { init(indexDao, Optional.of(threatSort)); this.index = index; - this.threatTriageField = triageLevelField; } public ElasticsearchMetaAlertDao() { @@ -196,7 +195,7 @@ public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) Document metaAlert = buildCreateDocument(alerts, request.getGroups()); calculateMetaScores(metaAlert); // Add source type to be consistent with other sources and allow filtering - metaAlert.getDocument().put(SOURCE_TYPE, MetaAlertDao.METAALERT_TYPE); + metaAlert.getDocument().put(getField(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE), MetaAlertDao.METAALERT_TYPE); // Start a list of updates / inserts we need to run Map> updates = new HashMap<>(); @@ -353,7 +352,7 @@ public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus statu List> currentAlerts = (List>) metaAlert.getDocument() .get(MetaAlertDao.ALERT_FIELD); currentAlerts.stream().forEach(currentAlert -> { - getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(SOURCE_TYPE))); + getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(getField(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE)))); }); Iterable alerts = indexDao.getAllLatest(getRequests); List> updatedAlerts = new ArrayList<>(); @@ -685,7 +684,7 @@ protected void calculateMetaScores(Document metaAlert) { ArrayList scores = new ArrayList<>(); for (Object alertRaw : alertsRaw) { Map alert = (Map) alertRaw; - Double scoreNum = parseThreatField(alert.get(threatTriageField)); + Double scoreNum = parseThreatField(alert.get(getField(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD))); if (scoreNum != null) { scores.add(scoreNum); } @@ -700,7 +699,7 @@ protected void calculateMetaScores(Document metaAlert) { Object threatScore = metaScores.getMetaScores().get(threatSort); // add the threat score as a float; type needs to match the threat score field from each of the sensor indices - metaAlert.getDocument().put(threatTriageField, ConversionUtils.convert(threatScore, Float.class)); + metaAlert.getDocument().put(getField(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD), ConversionUtils.convert(threatScore, Float.class)); } private Double parseThreatField(Object threatRaw) { @@ -720,4 +719,12 @@ public int getPageSize() { public void setPageSize(int pageSize) { this.pageSize = pageSize; } + + private String getField(String globalConfigKey, String defaultField) { + if (this.elasticsearchDao == null || this.elasticsearchDao.getAccessConfig() == null) { + return defaultField; + } + Map globalConfig = this.elasticsearchDao.getAccessConfig().getGlobalConfigSupplier().get(); + return (String) globalConfig.getOrDefault(globalConfigKey, defaultField); + } } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java index 1bfa9d61c7..44defb3129 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDaoTest.java @@ -214,10 +214,10 @@ public void testCalculateMetaScoresList() { List> alertList = new ArrayList<>(); // add an alert with a threat score - alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f)); + alertList.add( Collections.singletonMap(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, 10.0f)); // add a second alert with a threat score - alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 20.0f)); + alertList.add( Collections.singletonMap(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, 20.0f)); // add a third alert with NO threat score alertList.add( Collections.singletonMap("alert3", "has no threat score")); @@ -230,7 +230,7 @@ public void testCalculateMetaScoresList() { // calculate the threat score for the metaalert ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); metaAlertDao.calculateMetaScores(metaalert); - Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_FIELD_DEFAULT); + Object threatScore = metaalert.getDocument().get(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD); // the metaalert must contain a summary of all child threat scores assertEquals(20D, (Double) metaalert.getDocument().get("max"), delta); @@ -246,4 +246,31 @@ public void testCalculateMetaScoresList() { // by default, the overall threat score is the sum of all child threat scores assertEquals(30.0F, threatScore); } + + @Test + public void testCalculateMetaScoresWithDifferentFieldName() { + List> alertList = new ArrayList<>(); + + // add an alert with a threat score + alertList.add( Collections.singletonMap(MetaAlertDao.THREAT_FIELD_DEFAULT, 10.0f)); + + // create the metaalert + Map docMap = new HashMap<>(); + docMap.put(MetaAlertDao.ALERT_FIELD, alertList); + Document metaalert = new Document(docMap, "guid", MetaAlertDao.METAALERT_TYPE, 0L); + + // Configure a different threat triage score field name + AccessConfig accessConfig = new AccessConfig(); + accessConfig.setGlobalConfigSupplier(() -> new HashMap() {{ + put(MetaAlertDao.THREAT_FIELD_PROPERTY, MetaAlertDao.THREAT_FIELD_DEFAULT); + }}); + ElasticsearchDao elasticsearchDao = new ElasticsearchDao(); + elasticsearchDao.setAccessConfig(accessConfig); + + // calculate the threat score for the metaalert + ElasticsearchMetaAlertDao metaAlertDao = new ElasticsearchMetaAlertDao(); + metaAlertDao.init(elasticsearchDao); + metaAlertDao.calculateMetaScores(metaalert); + assertNotNull(metaalert.getDocument().get(MetaAlertDao.THREAT_FIELD_DEFAULT)); + } } diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java index 5222a38364..10522aaabc 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchMetaAlertIntegrationTest.java @@ -18,11 +18,14 @@ package org.apache.metron.elasticsearch.integration; +import static org.apache.metron.common.Constants.SENSOR_TYPE; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; import static org.apache.metron.indexing.dao.MetaAlertDao.ALERT_FIELD; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERTS_INDEX; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_FIELD; import static org.apache.metron.indexing.dao.MetaAlertDao.METAALERT_TYPE; import static org.apache.metron.indexing.dao.MetaAlertDao.STATUS_FIELD; +import static org.apache.metron.indexing.dao.MetaAlertDao.THREAT_FIELD_PROPERTY; import com.fasterxml.jackson.core.JsonProcessingException; import com.google.common.base.Joiner; @@ -85,7 +88,7 @@ public class ElasticsearchMetaAlertIntegrationTest { private static final String NEW_FIELD = "new-field"; private static final String NAME_FIELD = "name"; - private static IndexDao esDao; + private static ElasticsearchDao esDao; private static MetaAlertDao metaDao; private static ElasticSearchComponent es; @@ -193,6 +196,12 @@ public static void setupBefore() throws Exception { .withIndexDir(new File(INDEX_DIR)) .build(); es.start(); + } + + @Before + public void setup() throws IOException { + es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); + es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); AccessConfig accessConfig = new AccessConfig(); Map globalConfig = new HashMap() { @@ -212,12 +221,6 @@ public static void setupBefore() throws Exception { metaDao = new ElasticsearchMetaAlertDao(esDao); } - @Before - public void setup() throws IOException { - es.createIndexWithMapping(METAALERTS_INDEX, MetaAlertDao.METAALERT_DOC, template.replace("%MAPPING_NAME%", "metaalert")); - es.createIndexWithMapping(INDEX, "index_doc", template.replace("%MAPPING_NAME%", "index")); - } - @AfterClass public static void teardown() { if (es != null) { @@ -319,6 +322,12 @@ public void shouldCreateMetaAlert() throws Exception { // Verify metaAlert was created findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); } + { + // Verify metaalert has the default field names + Document metaAlert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); + Assert.assertTrue(metaAlert.getDocument().containsKey(ElasticsearchMetaAlertDao.SOURCE_TYPE)); + Assert.assertTrue(metaAlert.getDocument().containsKey(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD)); + } { // Verify alert 0 was not updated with metaalert field Document alert = metaDao.getLatest("message_0", SENSOR_NAME); @@ -342,6 +351,45 @@ public void shouldCreateMetaAlert() throws Exception { } } + @Test + public void shouldCreateMetaAlertWithConfiguredFieldNames() throws Exception { + // Configure field names + AccessConfig accessConfig = esDao.getAccessConfig(); + accessConfig.setGlobalConfigSupplier(() -> new HashMap() {{ + put("es.date.format", DATE_FORMAT); + put(SENSOR_TYPE_FIELD_PROPERTY, SENSOR_TYPE); + put(THREAT_FIELD_PROPERTY, MetaAlertDao.THREAT_FIELD_DEFAULT); + }}); + + // Load alerts + List> alerts = buildAlerts(1); + elasticsearchAdd(alerts, INDEX, SENSOR_NAME); + + // Verify load was successful + findCreatedDocs(Collections.singletonList( + new GetRequest("message_0", SENSOR_NAME))); + + { + MetaAlertCreateRequest metaAlertCreateRequest = new MetaAlertCreateRequest() {{ + setAlerts(new ArrayList() {{ + add(new GetRequest("message_0", SENSOR_NAME)); + }}); + setGroups(Collections.singletonList("group")); + }}; + MetaAlertCreateResponse metaAlertCreateResponse = metaDao.createMetaAlert(metaAlertCreateRequest); + { + // Verify metaAlert was created + findCreatedDoc(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); + } + { + // Verify alert 0 was not updated with metaalert field + Document metaAlert = metaDao.getLatest(metaAlertCreateResponse.getGuid(), MetaAlertDao.METAALERT_TYPE); + Assert.assertTrue(metaAlert.getDocument().containsKey(SENSOR_TYPE)); + Assert.assertTrue(metaAlert.getDocument().containsKey(MetaAlertDao.THREAT_FIELD_DEFAULT)); + } + } + } + @Test public void shouldAddAlertsToMetaAlert() throws Exception { // Load alerts @@ -1173,7 +1221,7 @@ protected List> buildAlerts(int count) { Map alerts = new HashMap<>(); alerts.put(Constants.GUID, guid); alerts.put("source:type", SENSOR_NAME); - alerts.put(MetaAlertDao.THREAT_FIELD_DEFAULT, i); + alerts.put(ElasticsearchMetaAlertDao.THREAT_TRIAGE_FIELD, i); alerts.put("timestamp", System.currentTimeMillis()); inputData.add(alerts); } diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java index 4530d2a5d2..6bc5f0084a 100644 --- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java +++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/MetaAlertDao.java @@ -68,7 +68,8 @@ public interface MetaAlertDao extends IndexDao { String METAALERT_TYPE = "metaalert"; String METAALERT_FIELD = "metaalerts"; String METAALERT_DOC = METAALERT_TYPE + "_doc"; - String THREAT_FIELD_DEFAULT = "threat:triage:score"; + String THREAT_FIELD_DEFAULT = "threat.triage.score"; + String THREAT_FIELD_PROPERTY = "threat.triage.score.field"; String THREAT_SORT_DEFAULT = "sum"; String ALERT_FIELD = "alert"; String STATUS_FIELD = "status"; From ff50445ee30701933abdda787379bafff0634028 Mon Sep 17 00:00:00 2001 From: merrimanr Date: Fri, 8 Jun 2018 11:48:11 -0500 Subject: [PATCH 2/2] moved getFieldName to metron-common --- .../rest/service/impl/SearchServiceImpl.java | 9 +-- .../service/impl/SearchServiceImplTest.java | 3 +- .../configuration/ConfigurationsUtils.java | 7 ++ .../dao/ElasticsearchMetaAlertDao.java | 69 +++++++++---------- 4 files changed, 44 insertions(+), 44 deletions(-) diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java index a001390ce2..1c92fcb858 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SearchServiceImpl.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Optional; import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.search.FieldType; import org.apache.metron.indexing.dao.search.GetRequest; @@ -147,13 +148,9 @@ public List getDefaultFacetFields() throws RestException { if (!alertUserSettings.isPresent() || alertUserSettings.get().getFacetFields() == null) { String facetFieldsProperty = environment .getProperty(SEARCH_FACET_FIELDS_SPRING_PROPERTY, String.class, ""); - - Map globalConfig = globalConfigService.get(); - String sourceTypeField = Constants.SENSOR_TYPE.replace('.', ':'); + String sourceTypeField = ConfigurationsUtils.getFieldName(globalConfigService.get(), SENSOR_TYPE_FIELD_PROPERTY, + Constants.SENSOR_TYPE.replace('.', ':')); List facetFields = new ArrayList<>(); - if (globalConfig != null) { - sourceTypeField = (String) globalConfig.getOrDefault(SENSOR_TYPE_FIELD_PROPERTY, sourceTypeField); - } facetFields.add(sourceTypeField); if (facetFieldsProperty != null) { facetFields.addAll(Arrays.asList(facetFieldsProperty.split(","))); diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java index 4c63c6d275..82e7221ad0 100644 --- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java +++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/SearchServiceImplTest.java @@ -17,6 +17,7 @@ */ package org.apache.metron.rest.service.impl; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; import static org.apache.metron.rest.MetronRestConstants.INDEX_WRITER_NAME; import static org.apache.metron.rest.MetronRestConstants.SEARCH_FACET_FIELDS_SPRING_PROPERTY; import static org.junit.Assert.assertEquals; @@ -183,7 +184,7 @@ public void testGetDefaultFacetFieldsGlobalConfig() throws RestException { when(environment.getProperty(SEARCH_FACET_FIELDS_SPRING_PROPERTY, String.class, "")) .thenReturn("ip_src_addr"); Map globalConfig = new HashMap<>(); - globalConfig.put("source.type.field", "source.type"); + globalConfig.put(SENSOR_TYPE_FIELD_PROPERTY, "source.type"); when(globalConfigService.get()).thenReturn(globalConfig); when(alertsUIService.getAlertsUIUserSettings()).thenReturn(Optional.empty()); List defaultFields = searchService.getDefaultFacetFields(); diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java index c7b39f0975..4550a41f82 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java @@ -799,5 +799,12 @@ public static void dumpConfigs(PrintStream out, CuratorFramework client, out.println(type + " Config: " + name + System.lineSeparator() + data); }, configType, configName); } + + public static String getFieldName(Map globalConfig, String globalConfigKey, String defaultFieldName) { + if (globalConfig == null) { + return defaultFieldName; + } + return (String) globalConfig.getOrDefault(globalConfigKey, defaultFieldName); + } } diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java index c3bb4a6f9d..ee3ca896e8 100644 --- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java @@ -18,22 +18,9 @@ package org.apache.metron.elasticsearch.dao; -import static org.apache.metron.common.Constants.GUID; -import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; -import static org.elasticsearch.index.query.QueryBuilders.boolQuery; -import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; -import static org.elasticsearch.index.query.QueryBuilders.existsQuery; -import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; -import static org.elasticsearch.index.query.QueryBuilders.termQuery; - -import com.fasterxml.jackson.databind.JsonNode; -import java.io.IOException; -import java.util.*; -import java.util.Map.Entry; -import java.util.stream.Collectors; -import org.apache.commons.collections4.SetUtils; import org.apache.lucene.search.join.ScoreMode; import org.apache.metron.common.Constants; +import org.apache.metron.common.configuration.ConfigurationsUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.MetaAlertDao; @@ -52,29 +39,37 @@ import org.apache.metron.indexing.dao.search.SearchResponse; import org.apache.metron.indexing.dao.search.SearchResult; import org.apache.metron.indexing.dao.update.Document; -import org.elasticsearch.action.get.GetResponse; -import org.elasticsearch.action.get.MultiGetItemResponse; -import org.elasticsearch.action.get.MultiGetRequest.Item; -import org.elasticsearch.action.get.MultiGetRequestBuilder; -import org.elasticsearch.action.get.MultiGetResponse; -import org.elasticsearch.action.index.IndexRequest; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.support.replication.ReplicationResponse.ShardInfo; -import org.elasticsearch.action.update.UpdateRequest; -import org.elasticsearch.action.update.UpdateResponse; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.InnerHitBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.index.query.QueryStringQueryBuilder; -import org.elasticsearch.search.SearchHit; import org.apache.metron.indexing.dao.update.OriginalNotFoundException; import org.apache.metron.indexing.dao.update.PatchRequest; import org.apache.metron.stellar.common.utils.ConversionUtils; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.index.query.InnerHitBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; +import org.elasticsearch.search.SearchHit; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.apache.metron.common.Constants.GUID; +import static org.apache.metron.common.Constants.SENSOR_TYPE_FIELD_PROPERTY; +import static org.elasticsearch.index.query.QueryBuilders.boolQuery; +import static org.elasticsearch.index.query.QueryBuilders.constantScoreQuery; +import static org.elasticsearch.index.query.QueryBuilders.existsQuery; +import static org.elasticsearch.index.query.QueryBuilders.nestedQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; public class ElasticsearchMetaAlertDao implements MetaAlertDao { @@ -195,7 +190,7 @@ public MetaAlertCreateResponse createMetaAlert(MetaAlertCreateRequest request) Document metaAlert = buildCreateDocument(alerts, request.getGroups()); calculateMetaScores(metaAlert); // Add source type to be consistent with other sources and allow filtering - metaAlert.getDocument().put(getField(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE), MetaAlertDao.METAALERT_TYPE); + metaAlert.getDocument().put(getFieldName(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE), MetaAlertDao.METAALERT_TYPE); // Start a list of updates / inserts we need to run Map> updates = new HashMap<>(); @@ -352,7 +347,7 @@ public boolean updateMetaAlertStatus(String metaAlertGuid, MetaAlertStatus statu List> currentAlerts = (List>) metaAlert.getDocument() .get(MetaAlertDao.ALERT_FIELD); currentAlerts.stream().forEach(currentAlert -> { - getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(getField(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE)))); + getRequests.add(new GetRequest((String) currentAlert.get(GUID), (String) currentAlert.get(getFieldName(SENSOR_TYPE_FIELD_PROPERTY, SOURCE_TYPE)))); }); Iterable alerts = indexDao.getAllLatest(getRequests); List> updatedAlerts = new ArrayList<>(); @@ -684,7 +679,7 @@ protected void calculateMetaScores(Document metaAlert) { ArrayList scores = new ArrayList<>(); for (Object alertRaw : alertsRaw) { Map alert = (Map) alertRaw; - Double scoreNum = parseThreatField(alert.get(getField(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD))); + Double scoreNum = parseThreatField(alert.get(getFieldName(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD))); if (scoreNum != null) { scores.add(scoreNum); } @@ -699,7 +694,7 @@ protected void calculateMetaScores(Document metaAlert) { Object threatScore = metaScores.getMetaScores().get(threatSort); // add the threat score as a float; type needs to match the threat score field from each of the sensor indices - metaAlert.getDocument().put(getField(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD), ConversionUtils.convert(threatScore, Float.class)); + metaAlert.getDocument().put(getFieldName(THREAT_FIELD_PROPERTY, THREAT_TRIAGE_FIELD), ConversionUtils.convert(threatScore, Float.class)); } private Double parseThreatField(Object threatRaw) { @@ -720,11 +715,11 @@ public void setPageSize(int pageSize) { this.pageSize = pageSize; } - private String getField(String globalConfigKey, String defaultField) { + private String getFieldName(String globalConfigKey, String defaultFieldName) { if (this.elasticsearchDao == null || this.elasticsearchDao.getAccessConfig() == null) { - return defaultField; + return defaultFieldName; } Map globalConfig = this.elasticsearchDao.getAccessConfig().getGlobalConfigSupplier().get(); - return (String) globalConfig.getOrDefault(globalConfigKey, defaultField); + return ConfigurationsUtils.getFieldName(globalConfig, globalConfigKey, defaultFieldName); } }