diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index cca6fab61..e1d730801 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -69,12 +69,12 @@ import org.opensearch.securityanalytics.resthandler.*; import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.action.SAIndexTIFSourceConfigAction; -import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.resthandler.RestGetTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.resthandler.RestIndexTIFSourceConfigAction; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.transport.TransportGetTIFSourceConfigAction; @@ -154,7 +154,7 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map private LogTypeService logTypeService; - private SATIFSourceConfigDao SaTifSourceConfigDao; + private SATIFSourceConfigService SaTifSourceConfigService; @Override public Collection getSystemIndexDescriptors(Settings settings){ @@ -192,8 +192,8 @@ public Collection createComponents(Client client, TIFJobParameterService tifJobParameterService = new TIFJobParameterService(client, clusterService); TIFJobUpdateService tifJobUpdateService = new TIFJobUpdateService(clusterService, tifJobParameterService, threatIntelFeedDataService, builtInTIFMetadataLoader); TIFLockService threatIntelLockService = new TIFLockService(clusterService, client); - SaTifSourceConfigDao = new SATIFSourceConfigDao(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); - SATIFSourceConfigService SaTifSourceConfigService = new SATIFSourceConfigService(SaTifSourceConfigDao, threatIntelLockService); + SaTifSourceConfigService = new SATIFSourceConfigService(client, clusterService, threadPool, xContentRegistry, threatIntelLockService); + SATIFSourceConfigManagementService SaTifSourceConfigManagementService = new SATIFSourceConfigManagementService(SaTifSourceConfigService, threatIntelLockService); TIFJobRunner.getJobRunnerInstance().initialize(clusterService, tifJobUpdateService, tifJobParameterService, threatIntelLockService, threadPool, detectorThreatIntelService); @@ -201,7 +201,7 @@ public Collection createComponents(Client client, return List.of( detectorIndices, correlationIndices, correlationRuleIndices, ruleTopicIndices, customLogTypeIndices, ruleIndices, mapperService, indexTemplateManager, builtinLogTypeLoader, builtInTIFMetadataLoader, threatIntelFeedDataService, detectorThreatIntelService, - tifJobUpdateService, tifJobParameterService, threatIntelLockService, SaTifSourceConfigDao, SaTifSourceConfigService); + tifJobUpdateService, tifJobParameterService, threatIntelLockService, SaTifSourceConfigService, SaTifSourceConfigManagementService); } @Override diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java deleted file mode 100644 index 3e3cfa311..000000000 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/dao/SATIFSourceConfigDao.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.securityanalytics.threatIntel.dao; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.opensearch.OpenSearchStatusException; -import org.opensearch.ResourceAlreadyExistsException; -import org.opensearch.action.StepListener; -import org.opensearch.action.admin.indices.create.CreateIndexRequest; -import org.opensearch.action.admin.indices.create.CreateIndexResponse; -import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; -import org.opensearch.action.index.IndexRequest; -import org.opensearch.action.index.IndexResponse; -import org.opensearch.action.support.WriteRequest; -import org.opensearch.client.Client; -import org.opensearch.cluster.service.ClusterService; -import org.opensearch.common.settings.ClusterSettings; -import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.xcontent.LoggingDeprecationHandler; -import org.opensearch.common.xcontent.XContentFactory; -import org.opensearch.common.xcontent.XContentHelper; -import org.opensearch.common.xcontent.XContentType; -import org.opensearch.core.action.ActionListener; -import org.opensearch.core.rest.RestStatus; -import org.opensearch.core.xcontent.NamedXContentRegistry; -import org.opensearch.core.xcontent.ToXContent; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; -import org.opensearch.securityanalytics.util.SecurityAnalyticsException; -import org.opensearch.threadpool.ThreadPool; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.StandardCharsets; -import java.util.stream.Collectors; - -/** - * CRUD for threat intel feeds source config object - */ -public class SATIFSourceConfigDao { - private static final Logger log = LogManager.getLogger(SATIFSourceConfigDao.class); - private final Client client; - private final ClusterService clusterService; - private final ClusterSettings clusterSettings; - private final ThreadPool threadPool; - private final NamedXContentRegistry xContentRegistry; - private final TIFLockService lockService; - - - public SATIFSourceConfigDao(final Client client, - final ClusterService clusterService, - ThreadPool threadPool, - NamedXContentRegistry xContentRegistry, - final TIFLockService lockService - ) { - this.client = client; - this.clusterService = clusterService; - this.clusterSettings = clusterService.getClusterSettings(); - this.threadPool = threadPool; - this.xContentRegistry = xContentRegistry; - this.lockService = lockService; - } - - public void indexTIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, - TimeValue indexTimeout, - final LockModel lock, - final ActionListener actionListener - ) { - StepListener createIndexStepListener = new StepListener<>(); - createIndexStepListener.whenComplete(v -> { - try { - IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) - .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) - .source(SaTifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) - .timeout(indexTimeout); - log.debug("Indexing tif source config"); - client.index(indexRequest, ActionListener.wrap(response -> { - log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); - SATIFSourceConfig responseSaTifSourceConfig = createSATIFSourceConfig(SaTifSourceConfig, response); - actionListener.onResponse(responseSaTifSourceConfig); - }, actionListener::onFailure)); - } catch (Exception e) { - log.error("Exception saving the threat intel source config in index", e); - actionListener.onFailure(e); - } - }, exception -> { - lockService.releaseLock(lock); - log.error("Failed to release lock", exception); - actionListener.onFailure(exception); - }); - createJobIndexIfNotExists(createIndexStepListener); - } - - private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, IndexResponse response) { - return new SATIFSourceConfig( - response.getId(), - SaTifSourceConfig.getVersion(), - SaTifSourceConfig.getName(), - SaTifSourceConfig.getFeedFormat(), - SaTifSourceConfig.getFeedType(), - SaTifSourceConfig.getCreatedByUser(), - SaTifSourceConfig.getCreatedAt(), - SaTifSourceConfig.getEnabledTime(), - SaTifSourceConfig.getLastUpdateTime(), - SaTifSourceConfig.getSchedule(), - SaTifSourceConfig.getState(), - SaTifSourceConfig.getRefreshType(), - SaTifSourceConfig.getLastRefreshedTime(), - SaTifSourceConfig.getLastRefreshedUser(), - SaTifSourceConfig.isEnabled(), - SaTifSourceConfig.getIocMapStore(), - SaTifSourceConfig.getIocTypes() - ); - } - - // Get the job config index mapping - private String getIndexMapping() { - try { - try (InputStream is = SATIFSourceConfigDao.class.getResourceAsStream("/mappings/threat_intel_job_mapping.json")) { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { - return reader.lines().map(String::trim).collect(Collectors.joining()); - } - } - } catch (IOException e) { - log.error("Failed to get the threat intel index mapping", e); - throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); - } - } - - // Create TIF source config index - /** - * Index name: .opensearch-sap--job - * Mapping: /mappings/threat_intel_job_mapping.json - * - * @param stepListener setup listener - */ - public void createJobIndexIfNotExists(final StepListener stepListener) { - // check if job index exists - if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { - stepListener.onResponse(null); - return; - } - final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) - .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); - StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { - @Override - public void onResponse(final CreateIndexResponse createIndexResponse) { - log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); - stepListener.onResponse(null); - } - - @Override - public void onFailure(final Exception e) { - if (e instanceof ResourceAlreadyExistsException) { - log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); - stepListener.onResponse(null); - return; - } - log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); - stepListener.onFailure(e); - } - })); - } - - - // Get TIF source config - public void getTIFSourceConfig( - String tifSourceConfigId, - Long version, - ActionListener actionListener - ) { - GetRequest getRequest = new GetRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME, tifSourceConfigId).version(version); - client.get(getRequest, new ActionListener<>() { - @Override - public void onResponse(GetResponse response) { - try { - if (!response.isExists()) { - actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND))); - return; - } - SATIFSourceConfig SaTifSourceConfig = null; - if (!response.isSourceEmpty()) { - XContentParser xcp = XContentHelper.createParser( - xContentRegistry, LoggingDeprecationHandler.INSTANCE, - response.getSourceAsBytesRef(), XContentType.JSON - ); - SaTifSourceConfig = SATIFSourceConfig.docParse(xcp, response.getId(), response.getVersion()); - assert SaTifSourceConfig != null; - } - log.debug("Threat intel source config with id [{}] fetched.", response.getId()); - actionListener.onResponse(SaTifSourceConfig); - } catch (IOException ex) { - log.error("Failed to fetch threat intel source config document", ex); - actionListener.onFailure(ex); - } - } - @Override - public void onFailure(Exception e) { - log.error("Failed to fetch threat intel source config document " + tifSourceConfigId, e); - actionListener.onFailure(e); - } - }); - } - -} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigManagementService.java similarity index 85% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigManagementService.java index 9f5438a6e..9824ff760 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigManagementService.java @@ -1,7 +1,7 @@ package org.opensearch.securityanalytics.threatIntel.sacommons; import org.opensearch.core.action.ActionListener; -public abstract class TIFSourceConfigService { +public abstract class TIFSourceConfigManagementService { IndexTIFSourceConfigResponse indexTIFConfig(IndexTIFSourceConfigRequest request, ActionListener listener){ return null; } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java new file mode 100644 index 000000000..c79577a4d --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigManagementService.java @@ -0,0 +1,119 @@ +package org.opensearch.securityanalytics.threatIntel.service; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.inject.Inject; +import org.opensearch.common.unit.TimeValue; +import org.opensearch.core.action.ActionListener; +import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; +import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; + +/** + * Service class for threat intel feed source config object + */ +public class SATIFSourceConfigManagementService { + private static final Logger log = LogManager.getLogger(SATIFSourceConfigManagementService.class); + private final SATIFSourceConfigService SaTifSourceConfigService; + private final TIFLockService lockService; + + /** + * Default constructor + * @param SaTifSourceConfigService the tif source config dao + * @param lockService the lock service + */ + @Inject + public SATIFSourceConfigManagementService( + final SATIFSourceConfigService SaTifSourceConfigService, + final TIFLockService lockService + ) { + this.SaTifSourceConfigService = SaTifSourceConfigService; + this.lockService = lockService; + } + + /** + * + * Creates the job index if it doesn't exist and indexes the tif source config object + * + * @param SaTifSourceConfigDto the tif source config dto + * @param lock the lock object + * @param indexTimeout the index time out + * @param listener listener that accepts a tif source config if successful + */ + public void createIndexAndSaveTIFSourceConfig( + final SATIFSourceConfigDto SaTifSourceConfigDto, + final LockModel lock, + final TimeValue indexTimeout, + final ActionListener listener + ) { + try { + SATIFSourceConfig SaTifSourceConfig = convertToSATIFConfig(SaTifSourceConfigDto); + SaTifSourceConfig.setState(TIFJobState.AVAILABLE); + SaTifSourceConfigService.indexTIFSourceConfig(SaTifSourceConfig, indexTimeout, lock, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig response) { + SaTifSourceConfig.setId(response.getId()); + SaTifSourceConfig.setVersion(response.getVersion()); + listener.onResponse(SaTifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + public void getTIFSourceConfig( + final String SaTifSourceConfigId, + final Long version, + final ActionListener listener + ) { + try { + SaTifSourceConfigService.getTIFSourceConfig(SaTifSourceConfigId, version, new ActionListener<>() { + @Override + public void onResponse(SATIFSourceConfig SaTifSourceConfig) { + listener.onResponse(SaTifSourceConfig); + } + @Override + public void onFailure(Exception e) { + listener.onFailure(e); + } + }); + } catch (Exception e) { + listener.onFailure(e); + } + } + + /** + * Converts the DTO to entity + * @param SaTifSourceConfigDto + * @return SaTifSourceConfig + */ + public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto SaTifSourceConfigDto) { + return new SATIFSourceConfig( + SaTifSourceConfigDto.getId(), + SaTifSourceConfigDto.getVersion(), + SaTifSourceConfigDto.getName(), + SaTifSourceConfigDto.getFeedFormat(), + SaTifSourceConfigDto.getFeedType(), + SaTifSourceConfigDto.getCreatedByUser(), + SaTifSourceConfigDto.getCreatedAt(), + SaTifSourceConfigDto.getEnabledTime(), + SaTifSourceConfigDto.getLastUpdateTime(), + SaTifSourceConfigDto.getSchedule(), + SaTifSourceConfigDto.getState(), + SaTifSourceConfigDto.getRefreshType(), + SaTifSourceConfigDto.getLastRefreshedTime(), + SaTifSourceConfigDto.getLastRefreshedUser(), + SaTifSourceConfigDto.isEnabled(), + SaTifSourceConfigDto.getIocMapStore(), + SaTifSourceConfigDto.getIocTypes() + ); + } + +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java index ec0dfb104..eab33adf9 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/SATIFSourceConfigService.java @@ -1,121 +1,216 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.opensearch.OpenSearchStatusException; +import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.action.StepListener; -import org.opensearch.common.inject.Inject; +import org.opensearch.action.admin.indices.create.CreateIndexRequest; +import org.opensearch.action.admin.indices.create.CreateIndexResponse; +import org.opensearch.action.get.GetRequest; +import org.opensearch.action.get.GetResponse; +import org.opensearch.action.index.IndexRequest; +import org.opensearch.action.index.IndexResponse; +import org.opensearch.action.support.WriteRequest; +import org.opensearch.client.Client; +import org.opensearch.cluster.service.ClusterService; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.unit.TimeValue; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.common.xcontent.XContentHelper; +import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.action.ActionListener; +import org.opensearch.core.rest.RestStatus; +import org.opensearch.core.xcontent.NamedXContentRegistry; +import org.opensearch.core.xcontent.ToXContent; import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; +import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.dao.SATIFSourceConfigDao; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; -import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; +import org.opensearch.securityanalytics.util.SecurityAnalyticsException; +import org.opensearch.threadpool.ThreadPool; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; /** - * Service class for threat intel feed source config object + * CRUD for threat intel feeds source config object */ public class SATIFSourceConfigService { private static final Logger log = LogManager.getLogger(SATIFSourceConfigService.class); - private final SATIFSourceConfigDao SaTifSourceConfigDao; + private final Client client; + private final ClusterService clusterService; + private final ClusterSettings clusterSettings; + private final ThreadPool threadPool; + private final NamedXContentRegistry xContentRegistry; private final TIFLockService lockService; - /** - * Default constructor - * @param SaTifSourceConfigDao the tif source config dao - * @param lockService the lock service - */ - @Inject - public SATIFSourceConfigService( - final SATIFSourceConfigDao SaTifSourceConfigDao, - final TIFLockService lockService + + public SATIFSourceConfigService(final Client client, + final ClusterService clusterService, + ThreadPool threadPool, + NamedXContentRegistry xContentRegistry, + final TIFLockService lockService ) { - this.SaTifSourceConfigDao = SaTifSourceConfigDao; + this.client = client; + this.clusterService = clusterService; + this.clusterSettings = clusterService.getClusterSettings(); + this.threadPool = threadPool; + this.xContentRegistry = xContentRegistry; this.lockService = lockService; } - /** - * - * Creates the job index if it doesn't exist and indexes the tif source config object - * - * @param SaTifSourceConfigDto the tif source config dto - * @param lock the lock object - * @param indexTimeout the index time out - * @param listener listener that accepts a tif source config if successful - */ - public void createIndexAndSaveTIFSourceConfig( - final SATIFSourceConfigDto SaTifSourceConfigDto, - final LockModel lock, - final TimeValue indexTimeout, - final ActionListener listener + public void indexTIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, + TimeValue indexTimeout, + final LockModel lock, + final ActionListener actionListener ) { - try { - SATIFSourceConfig SaTifSourceConfig = convertToSATIFConfig(SaTifSourceConfigDto); - SaTifSourceConfig.setState(TIFJobState.AVAILABLE); - SaTifSourceConfigDao.indexTIFSourceConfig(SaTifSourceConfig, indexTimeout, lock, new ActionListener<>() { - @Override - public void onResponse(SATIFSourceConfig response) { - SaTifSourceConfig.setId(response.getId()); - SaTifSourceConfig.setVersion(response.getVersion()); - listener.onResponse(SaTifSourceConfig); - } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); - } - }); - } catch (Exception e) { - listener.onFailure(e); - } + StepListener createIndexStepListener = new StepListener<>(); + createIndexStepListener.whenComplete(v -> { + try { + IndexRequest indexRequest = new IndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME) + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .source(SaTifSourceConfig.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS)) + .timeout(indexTimeout); + log.debug("Indexing tif source config"); + client.index(indexRequest, ActionListener.wrap(response -> { + log.debug("Threat intel source config with id [{}] indexed success.", response.getId()); + SATIFSourceConfig responseSaTifSourceConfig = createSATIFSourceConfig(SaTifSourceConfig, response); + actionListener.onResponse(responseSaTifSourceConfig); + }, actionListener::onFailure)); + } catch (Exception e) { + log.error("Exception saving the threat intel source config in index", e); + actionListener.onFailure(e); + } + }, exception -> { + lockService.releaseLock(lock); + log.error("Failed to release lock", exception); + actionListener.onFailure(exception); + }); + createJobIndexIfNotExists(createIndexStepListener); } - public void getTIFSourceConfig( - final String SaTifSourceConfigId, - final Long version, - final ActionListener listener - ) { + private static SATIFSourceConfig createSATIFSourceConfig(SATIFSourceConfig SaTifSourceConfig, IndexResponse response) { + return new SATIFSourceConfig( + response.getId(), + SaTifSourceConfig.getVersion(), + SaTifSourceConfig.getName(), + SaTifSourceConfig.getFeedFormat(), + SaTifSourceConfig.getFeedType(), + SaTifSourceConfig.getCreatedByUser(), + SaTifSourceConfig.getCreatedAt(), + SaTifSourceConfig.getEnabledTime(), + SaTifSourceConfig.getLastUpdateTime(), + SaTifSourceConfig.getSchedule(), + SaTifSourceConfig.getState(), + SaTifSourceConfig.getRefreshType(), + SaTifSourceConfig.getLastRefreshedTime(), + SaTifSourceConfig.getLastRefreshedUser(), + SaTifSourceConfig.isEnabled(), + SaTifSourceConfig.getIocMapStore(), + SaTifSourceConfig.getIocTypes() + ); + } + + // Get the job config index mapping + private String getIndexMapping() { try { - SaTifSourceConfigDao.getTIFSourceConfig(SaTifSourceConfigId, version, new ActionListener<>() { - @Override - public void onResponse(SATIFSourceConfig SaTifSourceConfig) { - listener.onResponse(SaTifSourceConfig); - } - @Override - public void onFailure(Exception e) { - listener.onFailure(e); + try (InputStream is = SATIFSourceConfigService.class.getResourceAsStream("/mappings/threat_intel_job_mapping.json")) { + try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) { + return reader.lines().map(String::trim).collect(Collectors.joining()); } - }); - } catch (Exception e) { - listener.onFailure(e); + } + } catch (IOException e) { + log.error("Failed to get the threat intel index mapping", e); + throw new SecurityAnalyticsException("Failed to get threat intel index mapping", RestStatus.INTERNAL_SERVER_ERROR, e); } } + // Create TIF source config index /** - * Converts the DTO to entity - * @param SaTifSourceConfigDto - * @return SaTifSourceConfig + * Index name: .opensearch-sap--job + * Mapping: /mappings/threat_intel_job_mapping.json + * + * @param stepListener setup listener */ - public SATIFSourceConfig convertToSATIFConfig(SATIFSourceConfigDto SaTifSourceConfigDto) { - return new SATIFSourceConfig( - SaTifSourceConfigDto.getId(), - SaTifSourceConfigDto.getVersion(), - SaTifSourceConfigDto.getName(), - SaTifSourceConfigDto.getFeedFormat(), - SaTifSourceConfigDto.getFeedType(), - SaTifSourceConfigDto.getCreatedByUser(), - SaTifSourceConfigDto.getCreatedAt(), - SaTifSourceConfigDto.getEnabledTime(), - SaTifSourceConfigDto.getLastUpdateTime(), - SaTifSourceConfigDto.getSchedule(), - SaTifSourceConfigDto.getState(), - SaTifSourceConfigDto.getRefreshType(), - SaTifSourceConfigDto.getLastRefreshedTime(), - SaTifSourceConfigDto.getLastRefreshedUser(), - SaTifSourceConfigDto.isEnabled(), - SaTifSourceConfigDto.getIocMapStore(), - SaTifSourceConfigDto.getIocTypes() - ); + public void createJobIndexIfNotExists(final StepListener stepListener) { + // check if job index exists + if (clusterService.state().metadata().hasIndex(SecurityAnalyticsPlugin.JOB_INDEX_NAME) == true) { + stepListener.onResponse(null); + return; + } + final CreateIndexRequest createIndexRequest = new CreateIndexRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME).mapping(getIndexMapping()) + .settings(SecurityAnalyticsPlugin.TIF_JOB_INDEX_SETTING); + StashedThreadContext.run(client, () -> client.admin().indices().create(createIndexRequest, new ActionListener<>() { + @Override + public void onResponse(final CreateIndexResponse createIndexResponse) { + log.debug("[{}] index created", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + } + + @Override + public void onFailure(final Exception e) { + if (e instanceof ResourceAlreadyExistsException) { + log.info("Index [{}] already exists", SecurityAnalyticsPlugin.JOB_INDEX_NAME); + stepListener.onResponse(null); + return; + } + log.error("Failed to create [{}] index", SecurityAnalyticsPlugin.JOB_INDEX_NAME, e); + stepListener.onFailure(e); + } + })); + } + + + // Get TIF source config + public void getTIFSourceConfig( + String tifSourceConfigId, + Long version, + ActionListener actionListener + ) { + GetRequest getRequest = new GetRequest(SecurityAnalyticsPlugin.JOB_INDEX_NAME, tifSourceConfigId).version(version); + client.get(getRequest, new ActionListener<>() { + @Override + public void onResponse(GetResponse response) { + try { + if (!response.isExists()) { + actionListener.onFailure(SecurityAnalyticsException.wrap(new OpenSearchStatusException("Threat intel source config not found.", RestStatus.NOT_FOUND))); + return; + } + SATIFSourceConfig SaTifSourceConfig = null; + if (!response.isSourceEmpty()) { + XContentParser xcp = XContentHelper.createParser( + xContentRegistry, LoggingDeprecationHandler.INSTANCE, + response.getSourceAsBytesRef(), XContentType.JSON + ); + SaTifSourceConfig = SATIFSourceConfig.docParse(xcp, response.getId(), response.getVersion()); + assert SaTifSourceConfig != null; + } + log.debug("Threat intel source config with id [{}] fetched.", response.getId()); + actionListener.onResponse(SaTifSourceConfig); + } catch (IOException ex) { + log.error("Failed to fetch threat intel source config document", ex); + actionListener.onFailure(ex); + } + } + @Override + public void onFailure(Exception e) { + log.error("Failed to fetch threat intel source config document " + tifSourceConfigId, e); + actionListener.onFailure(e); + } + }); } } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java index 93dd34ebc..a7512d2ac 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportGetTIFSourceConfigAction.java @@ -17,7 +17,7 @@ import org.opensearch.securityanalytics.threatIntel.action.SAGetTIFSourceConfigResponse; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfig; import org.opensearch.securityanalytics.threatIntel.model.SATIFSourceConfigDto; -import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigService; +import org.opensearch.securityanalytics.threatIntel.service.SATIFSourceConfigManagementService; import org.opensearch.securityanalytics.transport.SecureTransportAction; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; @@ -35,7 +35,7 @@ public class TransportGetTIFSourceConfigAction extends HandledTransportAction implements SecureTransportAction { private static final Logger log = LogManager.getLogger(TransportIndexTIFSourceConfigAction.class); - private final SATIFSourceConfigService SaTifSourceConfigService; + private final SATIFSourceConfigManagementService SaTifSourceConfigManagementService; private final TIFLockService lockService; private final ThreadPool threadPool; private final Settings settings; @@ -61,13 +59,13 @@ public TransportIndexTIFSourceConfigAction( final TransportService transportService, final ActionFilters actionFilters, final ThreadPool threadPool, - final SATIFSourceConfigService SaTifSourceConfigService, + final SATIFSourceConfigManagementService SaTifSourceConfigManagementService, final TIFLockService lockService, final Settings settings ) { super(SAIndexTIFSourceConfigAction.NAME, transportService, actionFilters, SAIndexTIFSourceConfigRequest::new); this.threadPool = threadPool; - this.SaTifSourceConfigService = SaTifSourceConfigService; + this.SaTifSourceConfigManagementService = SaTifSourceConfigManagementService; this.lockService = lockService; this.settings = settings; this.filterByEnabled = SecurityAnalyticsSettings.FILTER_BY_BACKEND_ROLES.get(this.settings); @@ -105,7 +103,7 @@ private void retrieveLockAndCreateTIFConfig(SAIndexTIFSourceConfigRequest reques SaTifSourceConfigDto.setCreatedByUser(user.getName()); } try { - SaTifSourceConfigService.createIndexAndSaveTIFSourceConfig(SaTifSourceConfigDto, + SaTifSourceConfigManagementService.createIndexAndSaveTIFSourceConfig(SaTifSourceConfigDto, lock, indexTimeout, new ActionListener<>() {