From 0e777709a2bbf3f63272d6bad14c9a7346b2baab Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 May 2024 13:30:53 -0700 Subject: [PATCH 1/4] refactored classes (#1037) Signed-off-by: Joanne Wang --- .../SecurityAnalyticsPlugin.java | 14 +++++++------- .../BuiltInTIFMetadataLoader.java | 2 +- .../jobscheduler/TIFJobRunner.java | 8 ++++---- .../TIFJobParameter.java | 3 +-- .../{common => model}/TIFMetadata.java | 2 +- .../DetectorThreatIntelService.java | 2 +- .../TIFJobParameterService.java | 7 ++----- .../TIFJobUpdateService.java | 9 ++++----- .../ThreatIntelFeedDataService.java | 10 ++++------ .../TransportPutTIFJobAction.java | 11 +++++++---- .../{ => util}/ThreatIntelFeedDataUtils.java | 2 +- .../{ => util}/ThreatIntelFeedParser.java | 4 ++-- .../TransportIndexDetectorAction.java | 2 +- .../TransportSearchDetectorAction.java | 2 +- .../SecurityAnalyticsRestTestCase.java | 2 +- .../threatIntel/ThreatIntelTestCase.java | 8 +++++--- .../action/TransportPutTIFJobActionTests.java | 19 ------------------- .../integTests/ThreatIntelJobRunnerIT.java | 4 ++-- .../jobscheduler/TIFJobParameterTests.java | 7 +++---- .../jobscheduler/TIFJobRunnerTests.java | 16 ---------------- 20 files changed, 48 insertions(+), 86 deletions(-) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{jobscheduler => model}/TIFJobParameter.java (99%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{common => model}/TIFMetadata.java (99%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => service}/DetectorThreatIntelService.java (99%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{jobscheduler => service}/TIFJobParameterService.java (96%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{jobscheduler => service}/TIFJobUpdateService.java (97%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => service}/ThreatIntelFeedDataService.java (97%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{action => transport}/TransportPutTIFJobAction.java (93%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => util}/ThreatIntelFeedDataUtils.java (96%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => util}/ThreatIntelFeedParser.java (93%) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index e7fe43106..652b438df 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -63,16 +63,16 @@ import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.*; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.securityanalytics.transport.*; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Detector; @@ -87,7 +87,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java index 6b84e9fe9..2b5856999 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java @@ -10,7 +10,7 @@ import org.opensearch.common.settings.SettingsException; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.util.FileUtils; import java.io.IOException; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index 1d8d8643f..65d7e46e5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -11,21 +11,21 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.time.Instant; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.threadpool.ThreadPool; /** diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java index bcbb84c1c..a964a1663 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.model; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; @@ -23,7 +23,6 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; import java.io.IOException; import java.time.Instant; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java index 04486fb7a..20035dcb8 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.common; +package org.opensearch.securityanalytics.threatIntel.model; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java index e541ee36c..6619b33f5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java index 55387cb35..c7fa5566e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,10 +15,8 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; @@ -30,11 +28,10 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexNotFoundException; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.BufferedReader; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java index 5c48ed8aa..eb90415b4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -15,16 +15,15 @@ import org.opensearch.OpenSearchStatusException; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.support.GroupedActionListener; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedParser; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java index b9d8aa3ea..61ea2374d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; @@ -20,7 +20,6 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; @@ -39,8 +38,8 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -51,7 +50,6 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,7 +58,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; /** * Service to handle CRUD operations on Threat Intel Feed Data diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java similarity index 93% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index a50beda35..c04c08798 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.action; +package org.opensearch.securityanalytics.threatIntel.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -19,11 +19,14 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; +import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java index a96558b50..20539695b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java similarity index 93% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index 92a66ed12..bfbb9dbde 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -12,7 +12,7 @@ import org.opensearch.SpecialPermission; import org.opensearch.common.SuppressForbidden; import org.opensearch.securityanalytics.threatIntel.common.Constants; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import java.io.BufferedReader; import java.io.IOException; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index ebe7d022d..f9fb9b198 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -97,7 +97,7 @@ import org.opensearch.securityanalytics.rules.backend.QueryBackend; import org.opensearch.securityanalytics.rules.exceptions.SigmaConditionError; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.ExceptionChecker; import org.opensearch.securityanalytics.util.IndexUtils; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java index 3b7b36503..5937769fe 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java @@ -19,7 +19,7 @@ import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.SearchDetectorRequest; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index e7da36705..0b5880bad 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -101,7 +101,7 @@ import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java index 20d36ab2d..d62ea5888 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java @@ -31,9 +31,11 @@ import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; import org.opensearch.test.client.NoOpNodeClient; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java index 27a01f5c0..f8c6ecadc 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java @@ -5,25 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.action; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import org.opensearch.action.StepListener; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.core.action.ActionListener; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.tasks.Task; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.util.ConcurrentModificationException; - -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - /*public class TransportPutTIFJobActionTests extends ThreatIntelTestCase { private TransportPutTIFJobAction action; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java index 1bf2025cd..03769ac43 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java @@ -28,7 +28,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; @@ -45,7 +45,7 @@ import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; public class ThreatIntelJobRunnerIT extends SecurityAnalyticsRestTestCase { private static final Logger log = LogManager.getLogger(ThreatIntelJobRunnerIT.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java index f7b7ff8d1..1d7f1706c 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java @@ -10,17 +10,16 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.TestHelpers; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.List; import java.util.Locale; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class TIFJobParameterTests extends ThreatIntelTestCase { private static final Logger log = LogManager.getLogger(TIFJobParameterTests.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java index 71bd68c61..ec13b7635 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java @@ -6,23 +6,7 @@ package org.opensearch.securityanalytics.threatIntel.jobscheduler; -import org.junit.Before; -import org.opensearch.jobscheduler.spi.JobDocVersion; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.time.Instant; -import java.util.Optional; - import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; /*public class TIFJobRunnerTests extends ThreatIntelTestCase { @Before From b2b156ee776cdeaa919607b13d4d4eb0713c17fa Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 May 2024 13:30:53 -0700 Subject: [PATCH 2/4] refactored classes (#1037) Signed-off-by: Joanne Wang --- .../SecurityAnalyticsPlugin.java | 14 +++++++------- .../BuiltInTIFMetadataLoader.java | 2 +- .../jobscheduler/TIFJobRunner.java | 8 ++++---- .../TIFJobParameter.java | 3 +-- .../{common => model}/TIFMetadata.java | 2 +- .../DetectorThreatIntelService.java | 2 +- .../TIFJobParameterService.java | 7 ++----- .../TIFJobUpdateService.java | 9 ++++----- .../ThreatIntelFeedDataService.java | 10 ++++------ .../TransportPutTIFJobAction.java | 11 +++++++---- .../{ => util}/ThreatIntelFeedDataUtils.java | 2 +- .../{ => util}/ThreatIntelFeedParser.java | 4 ++-- .../TransportIndexDetectorAction.java | 2 +- .../TransportSearchDetectorAction.java | 2 +- .../SecurityAnalyticsRestTestCase.java | 2 +- .../threatIntel/ThreatIntelTestCase.java | 8 +++++--- .../action/TransportPutTIFJobActionTests.java | 19 ------------------- .../integTests/ThreatIntelJobRunnerIT.java | 4 ++-- .../jobscheduler/TIFJobParameterTests.java | 7 +++---- .../jobscheduler/TIFJobRunnerTests.java | 16 ---------------- 20 files changed, 48 insertions(+), 86 deletions(-) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{jobscheduler => model}/TIFJobParameter.java (99%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{common => model}/TIFMetadata.java (99%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => service}/DetectorThreatIntelService.java (99%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{jobscheduler => service}/TIFJobParameterService.java (96%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{jobscheduler => service}/TIFJobUpdateService.java (97%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => service}/ThreatIntelFeedDataService.java (97%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{action => transport}/TransportPutTIFJobAction.java (93%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => util}/ThreatIntelFeedDataUtils.java (96%) rename src/main/java/org/opensearch/securityanalytics/threatIntel/{ => util}/ThreatIntelFeedParser.java (93%) diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index e7fe43106..652b438df 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -63,16 +63,16 @@ import org.opensearch.securityanalytics.model.CustomLogType; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.*; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobRunner; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.securityanalytics.transport.*; import org.opensearch.securityanalytics.model.Rule; import org.opensearch.securityanalytics.model.Detector; @@ -87,7 +87,7 @@ import org.opensearch.threadpool.ThreadPool; import org.opensearch.watcher.ResourceWatcherService; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, MapperPlugin, SearchPlugin, EnginePlugin, ClusterPlugin, SystemIndexPlugin, JobSchedulerExtension { diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java index 6b84e9fe9..2b5856999 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/feedMetadata/BuiltInTIFMetadataLoader.java @@ -10,7 +10,7 @@ import org.opensearch.common.settings.SettingsException; import org.opensearch.common.xcontent.XContentHelper; import org.opensearch.common.xcontent.json.JsonXContent; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.util.FileUtils; import java.io.IOException; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java index 1d8d8643f..65d7e46e5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunner.java @@ -11,21 +11,21 @@ import org.opensearch.cluster.service.ClusterService; import org.opensearch.core.action.ActionListener; import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; import org.opensearch.jobscheduler.spi.ScheduledJobParameter; import org.opensearch.jobscheduler.spi.ScheduledJobRunner; -import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import java.time.Instant; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.threadpool.ThreadPool; /** diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java index bcbb84c1c..a964a1663 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameter.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFJobParameter.java @@ -6,7 +6,7 @@ * this file be licensed under the Apache-2.0 license or a * compatible open source license. */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.model; import org.opensearch.core.ParseField; import org.opensearch.core.common.io.stream.StreamInput; @@ -23,7 +23,6 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; import java.io.IOException; import java.time.Instant; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java index 04486fb7a..20035dcb8 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFMetadata.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/TIFMetadata.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.common; +package org.opensearch.securityanalytics.threatIntel.model; import java.io.IOException; import java.util.Map; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java similarity index 99% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java index e541ee36c..6619b33f5 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/DetectorThreatIntelService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/DetectorThreatIntelService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java index 55387cb35..c7fa5566e 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobParameterService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,10 +15,8 @@ import org.opensearch.action.admin.indices.create.CreateIndexRequest; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.get.GetRequest; -import org.opensearch.action.get.GetResponse; import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; @@ -30,11 +28,10 @@ import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.ToXContent; import org.opensearch.core.xcontent.XContentParser; -import org.opensearch.index.IndexNotFoundException; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; import java.io.BufferedReader; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java index 5c48ed8aa..eb90415b4 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobUpdateService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/TIFJobUpdateService.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.jobscheduler; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVParser; import org.apache.commons.csv.CSVRecord; @@ -15,16 +15,15 @@ import org.opensearch.OpenSearchStatusException; import org.opensearch.action.admin.indices.create.CreateIndexResponse; import org.opensearch.action.support.GroupedActionListener; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.core.action.ActionListener; import org.opensearch.core.rest.RestStatus; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedParser; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedParser; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java similarity index 97% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java index b9d8aa3ea..61ea2374d 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataService.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/service/ThreatIntelFeedDataService.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.service; import org.apache.commons.csv.CSVRecord; import org.apache.commons.lang3.StringUtils; @@ -20,7 +20,6 @@ import org.opensearch.action.support.GroupedActionListener; import org.opensearch.action.support.IndicesOptions; import org.opensearch.action.support.WriteRequest; -import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.client.Client; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; @@ -39,8 +38,8 @@ import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.StashedThreadContext; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils; import org.opensearch.securityanalytics.util.IndexUtils; import org.opensearch.securityanalytics.util.SecurityAnalyticsException; @@ -51,7 +50,6 @@ import java.nio.charset.StandardCharsets; import java.time.Instant; import java.util.ArrayList; -import java.util.Collection; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -60,7 +58,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; /** * Service to handle CRUD operations on Threat Intel Feed Data diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java similarity index 93% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java index a50beda35..c04c08798 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobAction.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/transport/TransportPutTIFJobAction.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel.action; +package org.opensearch.securityanalytics.threatIntel.transport; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -19,11 +19,14 @@ import org.opensearch.core.rest.RestStatus; import org.opensearch.index.engine.VersionConflictEngineException; import org.opensearch.jobscheduler.spi.LockModel; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.action.PutTIFJobRequest; +import org.opensearch.securityanalytics.threatIntel.action.ThreatIntelIndicesResponse; import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; import org.opensearch.transport.TransportService; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java similarity index 96% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java index a96558b50..20539695b 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedDataUtils.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedDataUtils.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java similarity index 93% rename from src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java rename to src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java index 92a66ed12..bfbb9dbde 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelFeedParser.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/util/ThreatIntelFeedParser.java @@ -2,7 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ -package org.opensearch.securityanalytics.threatIntel; +package org.opensearch.securityanalytics.threatIntel.util; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVParser; @@ -12,7 +12,7 @@ import org.opensearch.SpecialPermission; import org.opensearch.common.SuppressForbidden; import org.opensearch.securityanalytics.threatIntel.common.Constants; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; import java.io.BufferedReader; import java.io.IOException; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java index ebe7d022d..f9fb9b198 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportIndexDetectorAction.java @@ -97,7 +97,7 @@ import org.opensearch.securityanalytics.rules.backend.QueryBackend; import org.opensearch.securityanalytics.rules.exceptions.SigmaConditionError; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.securityanalytics.util.ExceptionChecker; import org.opensearch.securityanalytics.util.IndexUtils; diff --git a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java index 3b7b36503..5937769fe 100644 --- a/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java +++ b/src/main/java/org/opensearch/securityanalytics/transport/TransportSearchDetectorAction.java @@ -19,7 +19,7 @@ import org.opensearch.securityanalytics.action.SearchDetectorAction; import org.opensearch.securityanalytics.action.SearchDetectorRequest; import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.action.TransportPutTIFJobAction; +import org.opensearch.securityanalytics.threatIntel.transport.TransportPutTIFJobAction; import org.opensearch.securityanalytics.util.DetectorIndices; import org.opensearch.tasks.Task; import org.opensearch.threadpool.ThreadPool; diff --git a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java index e7da36705..0b5880bad 100644 --- a/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/SecurityAnalyticsRestTestCase.java @@ -101,7 +101,7 @@ import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_MAX_DOCS; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_RETENTION_PERIOD; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.FINDING_HISTORY_ROLLOVER_PERIOD; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; import static org.opensearch.securityanalytics.util.RuleTopicIndices.ruleTopicIndexSettings; public class SecurityAnalyticsRestTestCase extends OpenSearchRestTestCase { diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java index 20d36ab2d..d62ea5888 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/ThreatIntelTestCase.java @@ -31,9 +31,11 @@ import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; import org.opensearch.securityanalytics.threatIntel.feedMetadata.BuiltInTIFMetadataLoader; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameterService; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobParameterService; +import org.opensearch.securityanalytics.threatIntel.service.TIFJobUpdateService; +import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; +import org.opensearch.securityanalytics.threatIntel.service.ThreatIntelFeedDataService; import org.opensearch.tasks.Task; import org.opensearch.tasks.TaskListener; import org.opensearch.test.client.NoOpNodeClient; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java index 27a01f5c0..f8c6ecadc 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/action/TransportPutTIFJobActionTests.java @@ -5,25 +5,6 @@ package org.opensearch.securityanalytics.threatIntel.action; -import org.junit.Before; -import org.mockito.ArgumentCaptor; -import org.opensearch.action.StepListener; -import org.opensearch.action.support.master.AcknowledgedResponse; -import org.opensearch.core.action.ActionListener; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; -import org.opensearch.tasks.Task; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.util.ConcurrentModificationException; - -import static org.mockito.ArgumentMatchers.*; -import static org.mockito.Mockito.*; - /*public class TransportPutTIFJobActionTests extends ThreatIntelTestCase { private TransportPutTIFJobAction action; diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java index 1bf2025cd..03769ac43 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/integTests/ThreatIntelJobRunnerIT.java @@ -28,7 +28,7 @@ import org.opensearch.securityanalytics.model.Detector; import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; -import org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; @@ -45,7 +45,7 @@ import static org.opensearch.securityanalytics.TestHelpers.*; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.ENABLE_WORKFLOW_USAGE; import static org.opensearch.securityanalytics.settings.SecurityAnalyticsSettings.TIF_UPDATE_INTERVAL; -import static org.opensearch.securityanalytics.threatIntel.ThreatIntelFeedDataUtils.getTifdList; +import static org.opensearch.securityanalytics.threatIntel.util.ThreatIntelFeedDataUtils.getTifdList; public class ThreatIntelJobRunnerIT extends SecurityAnalyticsRestTestCase { private static final Logger log = LogManager.getLogger(ThreatIntelJobRunnerIT.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java index f7b7ff8d1..1d7f1706c 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobParameterTests.java @@ -10,17 +10,16 @@ import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; import org.opensearch.securityanalytics.TestHelpers; -import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFMetadata; +import org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter; import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.List; import java.util.Locale; -import static org.opensearch.securityanalytics.threatIntel.jobscheduler.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; +import static org.opensearch.securityanalytics.threatIntel.model.TIFJobParameter.THREAT_INTEL_DATA_INDEX_NAME_PREFIX; public class TIFJobParameterTests extends ThreatIntelTestCase { private static final Logger log = LogManager.getLogger(TIFJobParameterTests.class); diff --git a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java index 71bd68c61..ec13b7635 100644 --- a/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java +++ b/src/test/java/org/opensearch/securityanalytics/threatIntel/jobscheduler/TIFJobRunnerTests.java @@ -6,23 +6,7 @@ package org.opensearch.securityanalytics.threatIntel.jobscheduler; -import org.junit.Before; -import org.opensearch.jobscheduler.spi.JobDocVersion; -import org.opensearch.jobscheduler.spi.JobExecutionContext; -import org.opensearch.jobscheduler.spi.LockModel; -import org.opensearch.jobscheduler.spi.ScheduledJobParameter; -import org.opensearch.securityanalytics.threatIntel.DetectorThreatIntelService; -import org.opensearch.securityanalytics.threatIntel.ThreatIntelTestCase; -import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; -import org.opensearch.securityanalytics.threatIntel.common.TIFLockService; -import org.opensearch.securityanalytics.TestHelpers; - -import java.io.IOException; -import java.time.Instant; -import java.util.Optional; - import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; /*public class TIFJobRunnerTests extends ThreatIntelTestCase { @Before From 76de7addaff3cff0bfb195a53373785346814db2 Mon Sep 17 00:00:00 2001 From: Surya Sashank Nistala Date: Tue, 21 May 2024 17:19:09 -0700 Subject: [PATCH 3/4] ioc match model (#1038) Signed-off-by: Surya Sashank Nistala --- .../securityanalytics/model/IoCMatch.java | 234 ++++++++++++++++++ .../resources/mappings/ioc_match_mapping.json | 38 +++ .../securityanalytics/TestHelpers.java | 7 + .../model/IoCMatchTests.java | 78 ++++++ 4 files changed, 357 insertions(+) create mode 100644 src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java create mode 100644 src/main/resources/mappings/ioc_match_mapping.json create mode 100644 src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java diff --git a/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java b/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java new file mode 100644 index 000000000..04f54699f --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IoCMatch.java @@ -0,0 +1,234 @@ +package org.opensearch.securityanalytics.model; + +import org.apache.commons.lang3.StringUtils; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import static org.opensearch.core.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * IoC Match provides mapping of the IoC Value to the list of docs that contain the ioc in a given execution of IoC_Scan_job + * It's the inverse of an IoC finding which maps a document to list of IoC's + */ +public class IoCMatch implements Writeable, ToXContent { + //TODO implement IoC_Match interface from security-analytics-commons + public static final String ID_FIELD = "id"; + public static final String RELATED_DOC_IDS_FIELD = "related_doc_ids"; + public static final String FEED_IDS_FIELD = "feed_ids"; + public static final String IOC_SCAN_JOB_ID_FIELD = "ioc_scan_job_id"; + public static final String IOC_SCAN_JOB_NAME_FIELD = "ioc_scan_job_name"; + public static final String IOC_VALUE_FIELD = "ioc_value"; + public static final String IOC_TYPE_FIELD = "ioc_type"; + public static final String TIMESTAMP_FIELD = "timestamp"; + public static final String EXECUTION_ID_FIELD = "execution_id"; + + private final String id; + private final List relatedDocIds; + private final List feedIds; + private final String iocScanJobId; + private final String iocScanJobName; + private final String iocValue; + private final String iocType; + private final Instant timestamp; + private final String executionId; + + public IoCMatch(String id, List relatedDocIds, List feedIds, String iocScanJobId, + String iocScanJobName, String iocValue, String iocType, Instant timestamp, String executionId) { + validateIoCMatch(id, iocScanJobId, iocScanJobName, iocValue, timestamp, executionId, relatedDocIds); + this.id = id; + this.relatedDocIds = relatedDocIds; + this.feedIds = feedIds; + this.iocScanJobId = iocScanJobId; + this.iocScanJobName = iocScanJobName; + this.iocValue = iocValue; + this.iocType = iocType; + this.timestamp = timestamp; + this.executionId = executionId; + } + + public IoCMatch(StreamInput in) throws IOException { + id = in.readString(); + relatedDocIds = in.readStringList(); + feedIds = in.readStringList(); + iocScanJobId = in.readString(); + iocScanJobName = in.readString(); + iocValue = in.readString(); + iocType = in.readString(); + timestamp = in.readInstant(); + executionId = in.readOptionalString(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeStringCollection(relatedDocIds); + out.writeStringCollection(feedIds); + out.writeString(iocScanJobId); + out.writeString(iocScanJobName); + out.writeString(iocValue); + out.writeString(iocType); + out.writeInstant(timestamp); + out.writeOptionalString(executionId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject() + .field(ID_FIELD, id) + .field(RELATED_DOC_IDS_FIELD, relatedDocIds) + .field(FEED_IDS_FIELD, feedIds) + .field(IOC_SCAN_JOB_ID_FIELD, iocScanJobId) + .field(IOC_SCAN_JOB_NAME_FIELD, iocScanJobName) + .field(IOC_VALUE_FIELD, iocValue) + .field(IOC_TYPE_FIELD, iocType) + .field(TIMESTAMP_FIELD, timestamp.toEpochMilli()) + .field(EXECUTION_ID_FIELD, executionId) + .endObject(); + return builder; + } + + public String getId() { + return id; + } + + public List getRelatedDocIds() { + return relatedDocIds; + } + + public List getFeedIds() { + return feedIds; + } + + public String getIocScanJobId() { + return iocScanJobId; + } + + public String getIocScanJobName() { + return iocScanJobName; + } + + public String getIocValue() { + return iocValue; + } + + public String getIocType() { + return iocType; + } + + public Instant getTimestamp() { + return timestamp; + } + + public String getExecutionId() { + return executionId; + } + + public static IoCMatch parse(XContentParser xcp) throws IOException { + String id = null; + List relatedDocIds = new ArrayList<>(); + List feedIds = new ArrayList<>(); + String iocScanJobId = null; + String iocScanName = null; + String iocValue = null; + String iocType = null; + Instant timestamp = null; + String executionId = null; + + ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case ID_FIELD: + id = xcp.text(); + break; + case RELATED_DOC_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + relatedDocIds.add(xcp.text()); + } + break; + case FEED_IDS_FIELD: + ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + feedIds.add(xcp.text()); + } + break; + case IOC_SCAN_JOB_ID_FIELD: + iocScanJobId = xcp.textOrNull(); + break; + case IOC_SCAN_JOB_NAME_FIELD: + iocScanName = xcp.textOrNull(); + break; + case IOC_VALUE_FIELD: + iocValue = xcp.textOrNull(); + break; + case IOC_TYPE_FIELD: + iocType = xcp.textOrNull(); + break; + case TIMESTAMP_FIELD: + try { + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + timestamp = null; + } else if (xcp.currentToken().isValue()) { + timestamp = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + timestamp = null; + } + break; + } catch (Exception e) { + throw new IllegalArgumentException("failed to parse timestamp in IoC Match object"); + } + case EXECUTION_ID_FIELD: + executionId = xcp.textOrNull(); + break; + } + } + + return new IoCMatch(id, relatedDocIds, feedIds, iocScanJobId, iocScanName, iocValue, iocType, timestamp, executionId); + } + + public static IoCMatch readFrom(StreamInput in) throws IOException { + return new IoCMatch(in); + } + + + private static void validateIoCMatch(String id, String iocScanJobId, String iocScanName, String iocValue, Instant timestamp, String executionId, List relatedDocIds) { + if (StringUtils.isBlank(id)) { + throw new IllegalArgumentException("id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocValue)) { + throw new IllegalArgumentException("ioc_value cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanJobId)) { + throw new IllegalArgumentException("ioc_scan_job_id cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(iocScanName)) { + throw new IllegalArgumentException("ioc_scan_job_name cannot be empty in IoC_Match Object"); + } + if (StringUtils.isBlank(executionId)) { + throw new IllegalArgumentException("execution_id cannot be empty in IoC_Match Object"); + } + if (timestamp == null) { + throw new IllegalArgumentException("timestamp cannot be null in IoC_Match Object"); + } + if(relatedDocIds == null || relatedDocIds.isEmpty()) { + throw new IllegalArgumentException("related_doc_ids cannot be null or empty in IoC_Match Object"); + } + } +} \ No newline at end of file diff --git a/src/main/resources/mappings/ioc_match_mapping.json b/src/main/resources/mappings/ioc_match_mapping.json new file mode 100644 index 000000000..f4573190e --- /dev/null +++ b/src/main/resources/mappings/ioc_match_mapping.json @@ -0,0 +1,38 @@ +{ + "dynamic": "strict", + "_meta" : { + "schema_version": 1 + }, + "properties": { + "schema_version": { + "type": "integer" + }, + "feed_ids" : { + "type": "keyword" + }, + "related_doc_ids": { + "type": "keyword" + }, + "ioc_scan_job_id": { + "type": "keyword" + }, + "ioc_scan_job_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "ioc_value" : { + "type": "keyword" + }, + "ioc_type" : { + "type": "keyword" + }, + "timestamp": { + "type": "long" + }, + "execution_id": { + "type": "keyword" + } + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index a1987138d..03dca9281 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -28,6 +28,7 @@ import org.opensearch.securityanalytics.model.DetectorInput; import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; +import org.opensearch.securityanalytics.model.IoCMatch; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -799,6 +800,12 @@ public static String toJsonStringWithUser(Detector detector) throws IOException return BytesReference.bytes(builder).utf8ToString(); } + public static String toJsonString(IoCMatch iocMatch) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = iocMatch.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + public static String toJsonString(ThreatIntelFeedData threatIntelFeedData) throws IOException { XContentBuilder builder = XContentFactory.jsonBuilder(); builder = threatIntelFeedData.toXContent(builder, ToXContent.EMPTY_PARAMS); diff --git a/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java new file mode 100644 index 000000000..4b56c7eb5 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IoCMatchTests.java @@ -0,0 +1,78 @@ +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.LoggingDeprecationHandler; +import org.opensearch.common.xcontent.XContentType; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; +import java.time.Instant; +import java.util.List; + +import static org.opensearch.securityanalytics.TestHelpers.toJsonString; + +public class IoCMatchTests extends OpenSearchTestCase { + + public void testIoCMatchAsAStream() throws IOException { + IoCMatch iocMatch = getRandomIoCMatch(); + String jsonString = toJsonString(iocMatch); + BytesStreamOutput out = new BytesStreamOutput(); + iocMatch.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IoCMatch newIocMatch = new IoCMatch(sin); + assertEquals(iocMatch.getId(), newIocMatch.getId()); + assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); + assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); + assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); + assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); + assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); + assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); + assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); + assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); + } + + public void testIoCMatchParse() throws IOException { + String iocMatchString = "{ \"id\": \"exampleId123\", \"related_doc_ids\": [\"relatedDocId1\", " + + "\"relatedDocId2\"], \"feed_ids\": [\"feedId1\", \"feedId2\"], \"ioc_scan_job_id\":" + + " \"scanJob123\", \"ioc_scan_job_name\": \"Example Scan Job\", \"ioc_value\": \"exampleIocValue\", " + + "\"ioc_type\": \"exampleIocType\", \"timestamp\": 1620912896000, \"execution_id\": \"execution123\" }"; + IoCMatch iocMatch = IoCMatch.parse((getParser(iocMatchString))); + BytesStreamOutput out = new BytesStreamOutput(); + iocMatch.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IoCMatch newIocMatch = new IoCMatch(sin); + assertEquals(iocMatch.getId(), newIocMatch.getId()); + assertEquals(iocMatch.getIocScanJobId(), newIocMatch.getIocScanJobId()); + assertEquals(iocMatch.getIocScanJobName(), newIocMatch.getIocScanJobName()); + assertEquals(iocMatch.getIocValue(), newIocMatch.getIocValue()); + assertEquals(iocMatch.getIocType(), newIocMatch.getIocType()); + assertEquals(iocMatch.getTimestamp(), newIocMatch.getTimestamp()); + assertEquals(iocMatch.getExecutionId(), newIocMatch.getExecutionId()); + assertTrue(iocMatch.getFeedIds().containsAll(newIocMatch.getFeedIds())); + assertTrue(iocMatch.getRelatedDocIds().containsAll(newIocMatch.getRelatedDocIds())); + } + + public XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent().createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } + + private static IoCMatch getRandomIoCMatch() { + return new IoCMatch( + randomAlphaOfLength(10), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + List.of(randomAlphaOfLength(10), randomAlphaOfLength(10)), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + randomAlphaOfLength(10), + Instant.now(), + randomAlphaOfLength(10)); + } + + +} From a99ca4d81109348a818768cbe9ea3aac2c06f48e Mon Sep 17 00:00:00 2001 From: Joanne Wang Date: Tue, 21 May 2024 18:31:54 -0700 Subject: [PATCH 4/4] Threat Intel Feed Config Model (#1028) --- .../threatIntel/common/FeedType.java | 21 + .../threatIntel/common/TIFJobState.java | 12 +- .../threatIntel/model/SATIFSourceConfig.java | 477 +++++++++++++++++ .../model/SATIFSourceConfigDto.java | 483 ++++++++++++++++++ .../sacommons/TIFSourceConfig.java | 66 +++ .../sacommons/TIFSourceConfigDto.java | 65 +++ .../mappings/threat_intel_job_mapping.json | 95 +++- 7 files changed, 1217 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedType.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java create mode 100644 src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedType.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedType.java new file mode 100644 index 000000000..606f9f1ec --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/FeedType.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.threatIntel.common; + +/** + * Types of feeds threat intel can support + * Feed types include: licensed, open-sourced, custom, and internal + */ +public enum FeedType { + + LICENSED, + + OPEN_SOURCED, + + CUSTOM, + + INTERNAL +} diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java index 22ffee3e9..db72ac757 100644 --- a/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/common/TIFJobState.java @@ -33,5 +33,15 @@ public enum TIFJobState { /** * tif job is being deleted */ - DELETING + DELETING, + + /** + * tif associated iocs are being refreshed + */ + REFRESHING, + + /** + * tif refresh job failed + */ + REFRESH_FAILED } diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java new file mode 100644 index 000000000..46f576b4e --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfig.java @@ -0,0 +1,477 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.UUIDs; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.jobscheduler.spi.ScheduledJobParameter; +import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule; +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfig; + +import java.io.IOException; +import java.time.Instant; +import java.util.Locale; +import java.util.Map; + +/** + * Implementation of TIF Config to store the feed configuration metadata and to schedule it onto the job scheduler + */ +public class SATIFSourceConfig implements TIFSourceConfig, Writeable, ScheduledJobParameter { + + private static final Logger log = LogManager.getLogger(SATIFSourceConfig.class); + + + /** + * Prefix of indices having threatIntel data + */ + public static final String THREAT_INTEL_DATA_INDEX_NAME_PREFIX = ".opensearch-sap-threat-intel"; + + public static final String NO_ID = ""; + public static final String ID_FIELD = "id"; + + public static final Long NO_VERSION = 1L; + public static final String VERSION_FIELD = "version"; + public static final String FEED_NAME_FIELD = "feed_name"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String FEED_TYPE_FIELD = "feed_type"; + public static final String CREATED_BY_USER_FIELD = "created_by_user"; + public static final String CREATED_AT_FIELD = "created_at"; + public static final String SOURCE_FIELD = "source"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String STATE_FIELD = "state"; + public static final String REFRESH_TYPE_FIELD = "refresh_type"; + public static final String LAST_REFRESHED_TIME_FIELD = "last_refreshed_time"; + public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; + public static final String ENABLED_FIELD = "enabled"; + public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + + private String id; + private Long version; + private String feedName; + private String feedFormat; + private FeedType feedType; + private String createdByUser; + private Instant createdAt; + + // private Source source; TODO: create Source Object + private Instant enabledTime; + private Instant lastUpdateTime; + private Schedule schedule; + private TIFJobState state; + public String refreshType; + public Instant lastRefreshedTime; + public String lastRefreshedUser; + private Boolean isEnabled; + private Map iocMapStore; + + public SATIFSourceConfig(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, + Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore) { + this.id = id != null ? id : NO_ID; + this.version = version != null ? version : NO_VERSION; + this.feedName = feedName; + this.feedFormat = feedFormat; + this.feedType = feedType; + this.createdByUser = createdByUser; + this.createdAt = createdAt != null ? createdAt : Instant.now(); + + if (this.isEnabled == null && this.enabledTime == null) { + this.enabledTime = Instant.now(); + } else if (this.isEnabled != null && !this.isEnabled) { + this.enabledTime = null; + } else { + this.enabledTime = enabledTime; + } + + this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); + this.schedule = schedule; + + this.state = (this.state == null) ? TIFJobState.CREATING : state; + + this.refreshType = refreshType; + this.lastRefreshedTime = lastRefreshedTime; + this.lastRefreshedUser = lastRefreshedUser; + this.isEnabled = isEnabled; + this.iocMapStore = iocMapStore; + } + + public SATIFSourceConfig(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readLong(), // version + sin.readString(), // feed name + sin.readString(), // feed format + FeedType.valueOf(sin.readString()), // feed type + sin.readString(), // created by user + sin.readInstant(), // created at + sin.readInstant(), // enabled time + sin.readInstant(), // last update time + new IntervalSchedule(sin), // schedule + TIFJobState.valueOf(sin.readString()), // state + sin.readString(), // refresh type + sin.readOptionalInstant(), // last refreshed time + sin.readOptionalString(), // last refreshed user + sin.readBoolean(), // is enabled + sin.readMap() // ioc map store + ); + } + + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeString(feedName); + out.writeString(feedFormat); + out.writeString(feedType.name()); + out.writeString(createdByUser); + out.writeInstant(createdAt); + out.writeInstant(enabledTime); + out.writeInstant(lastUpdateTime); + schedule.writeTo(out); + out.writeString(state.name()); + out.writeString(refreshType); + out.writeOptionalInstant(lastRefreshedTime == null ? null : lastRefreshedTime); + out.writeOptionalString(lastRefreshedUser == null? null : lastRefreshedUser); + out.writeBoolean(isEnabled); + out.writeMap(iocMapStore); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(ID_FIELD, id); + builder.field(VERSION_FIELD, version); + builder.field(FEED_NAME_FIELD, feedName); + builder.field(FEED_FORMAT_FIELD, feedFormat); + builder.field(FEED_TYPE_FIELD, feedType.name()); + builder.field(CREATED_BY_USER_FIELD, createdByUser); + + if (createdAt == null) { + builder.nullField(CREATED_AT_FIELD); + } else { + builder.timeField(CREATED_AT_FIELD, String.format(Locale.getDefault(), "%s_in_millis", CREATED_AT_FIELD), createdAt.toEpochMilli()); + } + + if (enabledTime == null) { + builder.nullField(ENABLED_TIME_FIELD); + } else { + builder.timeField(ENABLED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", ENABLED_TIME_FIELD), enabledTime.toEpochMilli()); + } + + if (lastUpdateTime == null) { + builder.nullField(LAST_UPDATE_TIME_FIELD); + } else { + builder.timeField(LAST_UPDATE_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_UPDATE_TIME_FIELD), lastUpdateTime.toEpochMilli()); + } + + builder.field(SCHEDULE_FIELD, schedule); + builder.field(STATE_FIELD, state.name()); + builder.field(REFRESH_TYPE_FIELD, refreshType); + if (lastRefreshedTime == null) { + builder.nullField(LAST_REFRESHED_TIME_FIELD); + } else { + builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", + LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); + } + builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); + builder.field(ENABLED_FIELD, isEnabled); + builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.endObject(); + return builder; + } + + public static SATIFSourceConfig parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = NO_ID; + } + if (version == null) { + version = NO_VERSION; + } + + String feedName = null; + String feedFormat = null; + FeedType feedType = null; + String createdByUser = null; + Instant createdAt = null; + Instant enabledTime = null; + Instant lastUpdateTime = null; + Schedule schedule = null; + TIFJobState state = null; + String refreshType = null; + Instant lastRefreshedTime = null; + String lastRefreshedUser = null; + Boolean isEnabled = null; + Map iocMapStore = null; + + xcp.nextToken(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case FEED_NAME_FIELD: + feedName = xcp.text(); + break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case FEED_TYPE_FIELD: + feedType = toFeedType(xcp.text()); + break; + case CREATED_BY_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdByUser = null; + } else { + createdByUser = xcp.text(); + } + break; + case CREATED_AT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdAt = null; + } else if (xcp.currentToken().isValue()) { + createdAt = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + createdAt = null; + } + break; + case ENABLED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + enabledTime = null; + } else if (xcp.currentToken().isValue()) { + enabledTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + enabledTime = null; + } + break; + case LAST_UPDATE_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + break; + case SCHEDULE_FIELD: + schedule = ScheduleParser.parse(xcp); + break; + case STATE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + state = TIFJobState.CREATING; + } else { + state = toState(xcp.text()); + } + break; + case REFRESH_TYPE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + refreshType = null; + } else { + refreshType = xcp.text(); + } + break; + case LAST_REFRESHED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedTime = null; + } else if (xcp.currentToken().isValue()) { + lastRefreshedTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastRefreshedTime = null; + } + break; + case LAST_REFRESHED_USER_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedUser = null; + } else { + lastRefreshedUser = xcp.text(); + } + break; + case ENABLED_FIELD: + isEnabled = xcp.booleanValue(); + break; + case IOC_MAP_STORE_FIELD: + iocMapStore = xcp.map(); + break; + + default: + xcp.skipChildren(); + } + } + + if (isEnabled && enabledTime == null) { + enabledTime = Instant.now(); + } else if (!isEnabled) { + enabledTime = null; + } + + return new SATIFSourceConfig( + id, + version, + feedName, + feedFormat, + feedType, + createdByUser, + createdAt != null ? createdAt : Instant.now(), + enabledTime, + lastUpdateTime != null ? lastUpdateTime : Instant.now(), + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocMapStore + ); + } + + + public static TIFJobState toState(String stateName) { + try { + return TIFJobState.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid state, cannot be parsed.", e); + return null; + } + } + + public static FeedType toFeedType(String feedType) { + try { + return FeedType.valueOf(feedType); + } catch (IllegalArgumentException e) { + log.error("Invalid feed type, cannot be parsed.", e); + return null; + } + } + + public static SATIFSourceConfig readFrom(StreamInput sin) throws IOException { + return new SATIFSourceConfig(sin); + } + + // Getters and Setters + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public Long getVersion() { + return version; + } + public void setVersion(Long version) { + this.version = version; + } + public String getName() { + return this.feedName; + } + public void setName(String name) { + this.feedName = name; + } + public String getFeedFormat() { + return feedFormat; + } + public void setFeedFormat(String feedFormat) { + this.feedFormat = feedFormat; + } + public FeedType getFeedType() { + return feedType; + } + public void setFeedType(FeedType feedType) { + this.feedType = feedType; + } + public String getCreatedByUser() { + return createdByUser; + } + public void setCreatedByUser(String createdByUser) { + this.createdByUser = createdByUser; + } + public Instant getCreatedAt() { + return createdAt; + } + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + public Instant getEnabledTime() { + return this.enabledTime; + } + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + public Schedule getSchedule() { + return this.schedule; + } + public void setSchedule(Schedule schedule) { + this.schedule = schedule; + } + public TIFJobState getState() { + return state; + } + public void setState(TIFJobState previousState) { + this.state = previousState; + } + public String getLastRefreshedUser() { + return lastRefreshedUser; + } + public void setLastRefreshedUser(String lastRefreshedUser) { + this.lastRefreshedUser = lastRefreshedUser; + } + public Instant getLastRefreshedTime() { + return lastRefreshedTime; + } + public void setLastRefreshedTime(Instant lastRefreshedTime) { + this.lastRefreshedTime = lastRefreshedTime; + } + public String getRefreshType() { + return refreshType; + } + public void setRefreshType(String refreshType) { + this.refreshType = refreshType; + } + public boolean isEnabled() { + return this.isEnabled; + } + public void enable() { + if (isEnabled == true) { + return; + } + enabledTime = Instant.now(); + isEnabled = true; + } + public void disable() { + enabledTime = null; + isEnabled = false; + } + public Map getIocMapStore() { + return iocMapStore; + } + public void setIocMapStore(Map iocMapStore) { + this.iocMapStore = iocMapStore; + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java new file mode 100644 index 000000000..c8344e5e1 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/model/SATIFSourceConfigDto.java @@ -0,0 +1,483 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ +package org.opensearch.securityanalytics.threatIntel.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.common.UUIDs; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.jobscheduler.spi.schedule.ScheduleParser; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; +import org.opensearch.securityanalytics.threatIntel.sacommons.TIFSourceConfigDto; + +import java.io.IOException; +import java.time.Instant; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +/** + * Implementation of TIF Config Dto to store the feed configuration metadata as DTO object + */ +public class SATIFSourceConfigDto implements Writeable, ToXContentObject, TIFSourceConfigDto { + + private static final Logger log = LogManager.getLogger(SATIFSourceConfigDto.class); + + + public static final String NO_ID = ""; + public static final String ID_FIELD = "id"; + + public static final Long NO_VERSION = 1L; + public static final String VERSION_FIELD = "version"; + public static final String FEED_NAME_FIELD = "feed_name"; + public static final String FEED_FORMAT_FIELD = "feed_format"; + public static final String FEED_TYPE_FIELD = "feed_type"; + public static final String CREATED_BY_USER_FIELD = "created_by_user"; + public static final String CREATED_AT_FIELD = "created_at"; + public static final String SOURCE_FIELD = "source"; + public static final String ENABLED_TIME_FIELD = "enabled_time"; + public static final String LAST_UPDATE_TIME_FIELD = "last_update_time"; + public static final String SCHEDULE_FIELD = "schedule"; + public static final String STATE_FIELD = "state"; + public static final String REFRESH_TYPE_FIELD = "refresh_type"; + public static final String LAST_REFRESHED_TIME_FIELD = "last_refreshed_time"; + public static final String LAST_REFRESHED_USER_FIELD = "last_refreshed_user"; + public static final String ENABLED_FIELD = "enabled"; + public static final String IOC_MAP_STORE_FIELD = "ioc_map_store"; + + private String id; + private Long version; + private String feedName; + private String feedFormat; + private FeedType feedType; + private String createdByUser; + private Instant createdAt; + + // private Source source; TODO: create Source Object + private Instant enabledTime; + private Instant lastUpdateTime; + private Schedule schedule; + private TIFJobState state; + public String refreshType; + public Instant lastRefreshedTime; + public String lastRefreshedUser; + private Boolean isEnabled; + private Map iocMapStore; + + public SATIFSourceConfigDto(SATIFSourceConfig saTIFSourceConfig) { + this.id = saTIFSourceConfig.getId(); + this.version = saTIFSourceConfig.getVersion(); + this.feedName = saTIFSourceConfig.getName(); + this.feedFormat = saTIFSourceConfig.getFeedFormat(); + this.feedType = saTIFSourceConfig.getFeedType(); + this.createdByUser = saTIFSourceConfig.getCreatedByUser(); + this.createdAt = saTIFSourceConfig.getCreatedAt(); + this.enabledTime = saTIFSourceConfig.getEnabledTime(); + this.lastUpdateTime = saTIFSourceConfig.getLastUpdateTime(); + this.schedule = saTIFSourceConfig.getSchedule(); + this.state = saTIFSourceConfig.getState();; + this.refreshType = saTIFSourceConfig.getRefreshType(); + this.lastRefreshedTime = saTIFSourceConfig.getLastRefreshedTime(); + this.lastRefreshedUser = saTIFSourceConfig.getLastRefreshedUser(); + this.isEnabled = saTIFSourceConfig.isEnabled();; + this.iocMapStore = saTIFSourceConfig.getIocMapStore(); + } + + public SATIFSourceConfigDto(String id, Long version, String feedName, String feedFormat, FeedType feedType, String createdByUser, Instant createdAt, + Instant enabledTime, Instant lastUpdateTime, Schedule schedule, TIFJobState state, String refreshType, Instant lastRefreshedTime, String lastRefreshedUser, + Boolean isEnabled, Map iocMapStore) { + this.id = id != null ? id : NO_ID; + this.version = version != null ? version : NO_VERSION; + this.feedName = feedName; + this.feedFormat = feedFormat; + this.feedType = feedType; + this.createdByUser = createdByUser; + this.createdAt = createdAt != null ? createdAt : Instant.now(); + + if (this.isEnabled == null && this.enabledTime == null) { + this.enabledTime = Instant.now(); + } else if (this.isEnabled != null && !this.isEnabled) { + this.enabledTime = null; + } else { + this.enabledTime = enabledTime; + } + + this.lastUpdateTime = lastUpdateTime != null ? lastUpdateTime : Instant.now(); + this.schedule = schedule; + + this.state = (this.state == null) ? TIFJobState.CREATING : state; + + this.refreshType = refreshType; + this.lastRefreshedTime = lastRefreshedTime; + this.lastRefreshedUser = lastRefreshedUser; + this.isEnabled = isEnabled; + this.iocMapStore = (this.iocMapStore == null) ? new HashMap<>() : iocMapStore; + } + + public SATIFSourceConfigDto(StreamInput sin) throws IOException { + this(new SATIFSourceConfig(sin)); + } + + public void writeTo(final StreamOutput out) throws IOException { + out.writeString(id); + out.writeLong(version); + out.writeString(feedName); + out.writeString(feedFormat); + out.writeString(feedType.name()); + out.writeString(createdByUser); + out.writeInstant(createdAt); + out.writeInstant(enabledTime); + out.writeInstant(lastUpdateTime); + schedule.writeTo(out); + out.writeString(state.name()); + out.writeOptionalString(refreshType == null? null: refreshType); + out.writeOptionalInstant(lastRefreshedTime == null ? null : lastRefreshedTime); + out.writeOptionalString(lastRefreshedUser == null? null : lastRefreshedUser); + out.writeBoolean(isEnabled); + out.writeMap(iocMapStore); + } + + @Override + public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException { + builder.startObject(); + builder.field(ID_FIELD, id); + builder.field(VERSION_FIELD, version); + builder.field(FEED_NAME_FIELD, feedName); + builder.field(FEED_FORMAT_FIELD, feedFormat); + builder.field(FEED_TYPE_FIELD, feedType); + builder.field(CREATED_BY_USER_FIELD, createdByUser); + + if (createdAt == null) { + builder.nullField(CREATED_AT_FIELD); + } else { + builder.timeField(CREATED_AT_FIELD, String.format(Locale.getDefault(), "%s_in_millis", CREATED_AT_FIELD), createdAt.toEpochMilli()); + } + + if (enabledTime == null) { + builder.nullField(ENABLED_TIME_FIELD); + } else { + builder.timeField(ENABLED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", ENABLED_TIME_FIELD), enabledTime.toEpochMilli()); + } + + if (lastUpdateTime == null) { + builder.nullField(LAST_UPDATE_TIME_FIELD); + } else { + builder.timeField(LAST_UPDATE_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", LAST_UPDATE_TIME_FIELD), lastUpdateTime.toEpochMilli()); + } + + builder.field(SCHEDULE_FIELD, schedule); + builder.field(STATE_FIELD, state.name()); + + if (refreshType == null) { + builder.nullField(REFRESH_TYPE_FIELD); + } else { + builder.field(REFRESH_TYPE_FIELD, refreshType); + } + + if (lastRefreshedTime == null) { + builder.nullField(LAST_REFRESHED_TIME_FIELD); + } else { + builder.timeField(LAST_REFRESHED_TIME_FIELD, String.format(Locale.getDefault(), "%s_in_millis", + LAST_REFRESHED_TIME_FIELD), lastRefreshedTime.toEpochMilli()); + } + + if (lastRefreshedUser == null) { + builder.nullField(LAST_REFRESHED_USER_FIELD); + } else { + builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); + } + builder.field(LAST_REFRESHED_USER_FIELD, lastRefreshedUser); + builder.field(ENABLED_FIELD, isEnabled); + builder.field(IOC_MAP_STORE_FIELD, iocMapStore); + builder.endObject(); + + return builder; + } + + public static SATIFSourceConfigDto parse(XContentParser xcp, String id, Long version) throws IOException { + if (id == null) { + id = NO_ID; + } + if (version == null) { + version = NO_VERSION; + } + + String feedName = null; + String feedFormat = null; + FeedType feedType = null; + String createdByUser = null; + Instant createdAt = null; + Instant enabledTime = null; + Instant lastUpdateTime = null; + Schedule schedule = null; + TIFJobState state = null; + String refreshType = null; + Instant lastRefreshedTime = null; + String lastRefreshedUser = null; + Boolean isEnabled = null; + Map iocMapStore = new HashMap<>(); + + xcp.nextToken(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case FEED_NAME_FIELD: + feedName = xcp.text(); + break; + case FEED_FORMAT_FIELD: + feedFormat = xcp.text(); + break; + case FEED_TYPE_FIELD: + feedType = toFeedType(xcp.text()); + break; + case CREATED_BY_USER_FIELD: + createdByUser = xcp.text(); + break; + case CREATED_AT_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + createdAt = null; + } else if (xcp.currentToken().isValue()) { + createdAt = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + createdAt = null; + } + break; + case ENABLED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + enabledTime = null; + } else if (xcp.currentToken().isValue()) { + enabledTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + enabledTime = null; + } + break; + case LAST_UPDATE_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastUpdateTime = null; + } else if (xcp.currentToken().isValue()) { + lastUpdateTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastUpdateTime = null; + } + break; + case SCHEDULE_FIELD: + schedule = ScheduleParser.parse(xcp); + break; + case STATE_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + state = TIFJobState.CREATING; + } else { + state = toState(xcp.text()); + } + break; + case REFRESH_TYPE_FIELD: + refreshType = xcp.text(); + break; + case LAST_REFRESHED_TIME_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + lastRefreshedTime = null; + } else if (xcp.currentToken().isValue()) { + lastRefreshedTime = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + lastRefreshedTime = null; + } + break; + case LAST_REFRESHED_USER_FIELD: + lastRefreshedUser = xcp.text(); + break; + case ENABLED_FIELD: + isEnabled = xcp.booleanValue(); + break; + case IOC_MAP_STORE_FIELD: + iocMapStore = xcp.map(); + break; + + default: + xcp.skipChildren(); + } + } + + if (isEnabled && enabledTime == null) { + enabledTime = Instant.now(); + } else if (!isEnabled) { + enabledTime = null; + } + + return new SATIFSourceConfigDto( + id, + version, + feedName, + feedFormat, + feedType, + createdByUser, + createdAt != null ? createdAt : Instant.now(), + enabledTime, + lastUpdateTime != null ? lastUpdateTime : Instant.now(), + schedule, + state, + refreshType, + lastRefreshedTime, + lastRefreshedUser, + isEnabled, + iocMapStore + ); + } + + // TODO: refactor out to sa commons + public static TIFJobState toState(String stateName) { + try { + return TIFJobState.valueOf(stateName); + } catch (IllegalArgumentException e) { + log.error("Invalid state, cannot be parsed.", e); + return null; + } + } + + public static FeedType toFeedType(String feedType) { + try { + return FeedType.valueOf(feedType); + } catch (IllegalArgumentException e) { + log.error("Invalid feed type, cannot be parsed.", e); + return null; + } + } + + + // Getters and Setters + public String getId() { + return id; + } + public void setId(String id) { + this.id = id; + } + public Long getVersion() { + return version; + } + public void setVersion(Long version) { + this.version = version; + } + public String getName() { + return this.feedName; + } + public void setName(String name) { + this.feedName = name; + } + public String getFeedFormat() { + return feedFormat; + } + public void setFeedFormat(String feedFormat) { + this.feedFormat = feedFormat; + } + public FeedType getFeedType() { + return feedType; + } + public void setFeedType(FeedType feedType) { + this.feedType = feedType; + } + public String getCreatedByUser() { + return createdByUser; + } + public void setCreatedByUser(String createdByUser) { + this.createdByUser = createdByUser; + } + public Instant getCreatedAt() { + return createdAt; + } + public void setCreatedAt(Instant createdAt) { + this.createdAt = createdAt; + } + public Instant getEnabledTime() { + return this.enabledTime; + } + public void setEnabledTime(Instant enabledTime) { + this.enabledTime = enabledTime; + } + public Instant getLastUpdateTime() { + return this.lastUpdateTime; + } + public void setLastUpdateTime(Instant lastUpdateTime) { + this.lastUpdateTime = lastUpdateTime; + } + public Schedule getSchedule() { + return this.schedule; + } + public void setSchedule(Schedule schedule) { + this.schedule = schedule; + } + public TIFJobState getState() { + return state; + } + public void setState(TIFJobState previousState) { + this.state = previousState; + } + public String getLastRefreshedUser() { + return lastRefreshedUser; + } + public void setLastRefreshedUser(String lastRefreshedUser) { + this.lastRefreshedUser = lastRefreshedUser; + } + public Instant getLastRefreshedTime() { + return lastRefreshedTime; + } + public void setLastRefreshedTime(Instant lastRefreshedTime) { + this.lastRefreshedTime = lastRefreshedTime; + } + public String getRefreshType() { + return refreshType; + } + public void setRefreshType(String refreshType) { + this.refreshType = refreshType; + } + public boolean isEnabled() { + return this.isEnabled; + } + + /** + * Enable auto update of threat intel feed data + */ + public void enable() { + if (isEnabled == true) { + return; + } + enabledTime = Instant.now(); + isEnabled = true; + } + + /** + * Disable auto update of threat intel feed data + */ + public void disable() { + enabledTime = null; + isEnabled = false; + } + public Map getIocMapStore() { + return iocMapStore; + } + public void setIocMapStore(Map iocMapStore) { + this.iocMapStore = iocMapStore; + } + public static SATIFSourceConfigDto readFrom(StreamInput sin) throws IOException { + return new SATIFSourceConfigDto(sin); + } +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java new file mode 100644 index 000000000..847fb3be9 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfig.java @@ -0,0 +1,66 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; + +import java.time.Instant; +import java.util.Map; + +/** + * Threat intel config interface + */ +public interface TIFSourceConfig { + String getId(); + + void setId(String id); + + Long getVersion(); + + void setVersion(Long version); + + String getName(); + + void setName(String feedName); + + String getFeedFormat(); + + void setFeedFormat(String feedFormat); + + FeedType getFeedType(); + + void setFeedType(FeedType feedType); + + String getCreatedByUser(); + + void setCreatedByUser(String createdByUser); + + Instant getCreatedAt(); + + void setCreatedAt(Instant createdAt); + + Instant getEnabledTime(); + + void setEnabledTime(Instant enabledTime); + + Instant getLastUpdateTime(); + + void setLastUpdateTime(Instant lastUpdateTime); + + Schedule getSchedule(); + + void setSchedule(Schedule schedule); + + TIFJobState getState(); + + void setState(TIFJobState previousState); + + void enable(); + + void disable(); + + Map getIocMapStore(); + + void setIocMapStore(Map iocMapStore); + +} \ No newline at end of file diff --git a/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java new file mode 100644 index 000000000..c8e27d1fa --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/threatIntel/sacommons/TIFSourceConfigDto.java @@ -0,0 +1,65 @@ +package org.opensearch.securityanalytics.threatIntel.sacommons; + +import org.opensearch.jobscheduler.spi.schedule.Schedule; +import org.opensearch.securityanalytics.threatIntel.common.FeedType; +import org.opensearch.securityanalytics.threatIntel.common.TIFJobState; + +import java.time.Instant; +import java.util.Map; + +/** + * Threat intel config dto interface + */ +public interface TIFSourceConfigDto { + String getId(); + + void setId(String id); + + Long getVersion(); + + void setVersion(Long version); + + String getName(); + + void setName(String feedName); + + String getFeedFormat(); + + void setFeedFormat(String feedFormat); + + FeedType getFeedType(); + + void setFeedType(FeedType feedType); + + String getCreatedByUser(); + + void setCreatedByUser(String createdByUser); + + Instant getCreatedAt(); + + void setCreatedAt(Instant createdAt); + + Instant getEnabledTime(); + + void setEnabledTime(Instant enabledTime); + + Instant getLastUpdateTime(); + + void setLastUpdateTime(Instant lastUpdateTime); + + Schedule getSchedule(); + + void setSchedule(Schedule schedule); + + TIFJobState getState(); + + void setState(TIFJobState previousState); + + void enable(); + + void disable(); + + Map getIocMapStore(); + + void setIocMapStore(Map iocMapStore); +} \ No newline at end of file diff --git a/src/main/resources/mappings/threat_intel_job_mapping.json b/src/main/resources/mappings/threat_intel_job_mapping.json index ffd165ae5..59d49f73d 100644 --- a/src/main/resources/mappings/threat_intel_job_mapping.json +++ b/src/main/resources/mappings/threat_intel_job_mapping.json @@ -1,9 +1,102 @@ { "dynamic": "strict", "_meta" : { - "schema_version": 1 + "schema_version": 2 }, "properties": { + "feed_format_config": { + "dynamic": "false", + "properties": { + "feed_name": { + "type" : "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + }, + "feed_format": { + "type": "keyword" + }, + "feed_type": { + "type": "text" + }, + "created_by_user": { + "type": "keyword" + }, + "created_at": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "source" : { + "type": "nested", + "properties": { + "type": { + "type": "keyword" + }, + "url": { + "type": "keyword" + }, + "path": { + "type": "text" + }, + "security": { + "type": "text", + "fields" : { + "keyword" : { + "type" : "keyword" + } + } + } + } + }, + "enabled_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "last_update_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "schedule": { + "properties": { + "interval": { + "properties": { + "period": { + "type": "integer" + }, + "start_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "unit": { + "type": "keyword" + } + } + } + } + }, + "state": { + "type": "text" + }, + "refresh_type": { + "type": "keyword" + }, + "last_refreshed_time": { + "type": "date", + "format": "strict_date_time||epoch_millis" + }, + "last_refreshed_user": { + "type": "keyword" + }, + "enabled": { + "type": "boolean" + }, + "version": { + "type": "long" + } + } + }, "schema_version": { "type": "integer" },