diff --git a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java index 652b438df..534cd04d9 100644 --- a/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java +++ b/src/main/java/org/opensearch/securityanalytics/SecurityAnalyticsPlugin.java @@ -7,6 +7,7 @@ import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.function.Supplier; import java.util.Optional; @@ -61,6 +62,7 @@ import org.opensearch.securityanalytics.mapper.IndexTemplateManager; import org.opensearch.securityanalytics.mapper.MapperService; import org.opensearch.securityanalytics.model.CustomLogType; +import org.opensearch.securityanalytics.model.IocDao; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.securityanalytics.resthandler.*; import org.opensearch.securityanalytics.threatIntel.service.DetectorThreatIntelService; @@ -103,10 +105,17 @@ public class SecurityAnalyticsPlugin extends Plugin implements ActionPlugin, Map public static final String FINDINGS_CORRELATE_URI = FINDINGS_BASE_URI + "/correlate"; public static final String LIST_CORRELATIONS_URI = PLUGINS_BASE_URI + "/correlations"; public static final String CORRELATION_RULES_BASE_URI = PLUGINS_BASE_URI + "/correlation/rules"; + public static final String IOC_BASE_URI = PLUGINS_BASE_URI + "/ioc"; + public static final String IOC_FETCH_BASE_URI = IOC_BASE_URI + "/fetch"; public static final String CUSTOM_LOG_TYPE_URI = PLUGINS_BASE_URI + "/logtype"; public static final String JOB_INDEX_NAME = ".opensearch-sap--job"; public static final Map TIF_JOB_INDEX_SETTING = Map.of(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1, IndexMetadata.SETTING_AUTO_EXPAND_REPLICAS, "0-all", IndexMetadata.SETTING_INDEX_HIDDEN, true); + public static final String IOC_INDEX_NAME_BASE = ".opensearch-sap-ioc"; + public static final String IOC_ALL_INDEX_PATTERN = IOC_INDEX_NAME_BASE + "-*"; + public static final String IOC_DOMAIN_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.DOMAIN.name().toLowerCase(Locale.ROOT); + public static final String IOC_HASH_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.HASH.name().toLowerCase(Locale.ROOT); + public static final String IOC_IP_INDEX_NAME = IOC_INDEX_NAME_BASE + IocDao.IocType.IP.name().toLowerCase(Locale.ROOT); private CorrelationRuleIndices correlationRuleIndices; diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocDao.java b/src/main/java/org/opensearch/securityanalytics/model/IocDao.java new file mode 100644 index 000000000..6719af006 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IocDao.java @@ -0,0 +1,334 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_DOMAIN_INDEX_NAME; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_HASH_INDEX_NAME; +import static org.opensearch.securityanalytics.SecurityAnalyticsPlugin.IOC_IP_INDEX_NAME; + +public class IocDao implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(IocDao.class); + + public static final String NO_ID = ""; + + static final String ID_FIELD = "id"; + static final String NAME_FIELD = "name"; + static final String TYPE_FIELD = "type"; + static final String VALUE_FIELD = "value"; + static final String SEVERITY_FIELD = "severity"; + static final String SPEC_VERSION_FIELD = "spec_version"; + static final String CREATED_FIELD = "created"; + static final String MODIFIED_FIELD = "modified"; + static final String DESCRIPTION_FIELD = "description"; + static final String LABELS_FIELD = "labels"; + static final String FEED_ID_FIELD = "feed_id"; + + private String id; + private String name; + private IocType type; + private String value; + private String severity; + private String specVersion; + private Instant created; + private Instant modified; + private String description; + private List labels; + private String feedId; + + public IocDao( + String id, + String name, + IocType type, + String value, + String severity, + String specVersion, + Instant created, + Instant modified, + String description, + List labels, + String feedId + ) { + this.id = id == null ? NO_ID : id; + this.name = name; + this.type = type; + this.value = value; + this.severity = severity; + this.specVersion = specVersion; + this.created = created; + this.modified = modified; + this.description = description; + this.labels = labels == null ? Collections.emptyList() : labels; + this.feedId = feedId; + validate(); + } + + public IocDao(StreamInput sin) throws IOException { + this( + sin.readString(), // id + sin.readString(), // name + sin.readEnum(IocType.class), // type + sin.readString(), // value + sin.readString(), // severity + sin.readString(), // specVersion + sin.readInstant(), // created + sin.readInstant(), // modified + sin.readString(), // description + sin.readStringList(), // labels + sin.readString() // feedId + ); + } + + public IocDao(IocDto iocDto) { + this( + iocDto.getId(), + iocDto.getName(), + iocDto.getType(), + iocDto.getValue(), + iocDto.getSeverity(), + iocDto.getSpecVersion(), + iocDto.getCreated(), + iocDto.getModified(), + iocDto.getDescription(), + iocDto.getLabels(), + iocDto.getFeedId() + ); + } + + public static IocDao readFrom(StreamInput sin) throws IOException { + return new IocDao(sin); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(name); + out.writeEnum(type); + out.writeString(value); + out.writeString(severity); + out.writeString(specVersion); + out.writeInstant(created); + out.writeInstant(modified); + out.writeString(description); + out.writeStringCollection(labels); + out.writeString(feedId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(ID_FIELD, id) + .field(NAME_FIELD, name) + .field(TYPE_FIELD, type) + .field(VALUE_FIELD, value) + .field(SEVERITY_FIELD, severity) + .field(SPEC_VERSION_FIELD, specVersion) + .timeField(CREATED_FIELD, created) + .timeField(MODIFIED_FIELD, modified) + .field(DESCRIPTION_FIELD, description) + .field(LABELS_FIELD, labels) + .field(FEED_ID_FIELD, feedId) + .endObject(); + } + + public static IocDao parse(XContentParser xcp, String id) throws IOException { + if (id == null) { + id = NO_ID; + } + + String name = null; + IocType type = null; + String value = null; + String severity = null; + String specVersion = null; + Instant created = null; + Instant modified = null; + String description = null; + List labels = Collections.emptyList(); + String feedId = null; + + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_OBJECT) { + String fieldName = xcp.currentName(); + xcp.nextToken(); + + switch (fieldName) { + case NAME_FIELD: + name = xcp.text(); + break; + case TYPE_FIELD: + type = IocType.valueOf(xcp.text().toUpperCase(Locale.ROOT)); + break; + case VALUE_FIELD: + value = xcp.text(); + break; + case SEVERITY_FIELD: + severity = xcp.text(); + break; + case SPEC_VERSION_FIELD: + specVersion = xcp.text(); + break; + case CREATED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + created = null; + } else if (xcp.currentToken().isValue()) { + created = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + created = null; + } + break; + case MODIFIED_FIELD: + if (xcp.currentToken() == XContentParser.Token.VALUE_NULL) { + modified = null; + } else if (xcp.currentToken().isValue()) { + modified = Instant.ofEpochMilli(xcp.longValue()); + } else { + XContentParserUtils.throwUnknownToken(xcp.currentToken(), xcp.getTokenLocation()); + modified = null; + } + break; + case DESCRIPTION_FIELD: + description = xcp.text(); + break; + case LABELS_FIELD: + XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_ARRAY, xcp.currentToken(), xcp); + while (xcp.nextToken() != XContentParser.Token.END_ARRAY) { + String entry = xcp.textOrNull(); + if (entry != null) { + labels.add(entry); + } + } + break; + case FEED_ID_FIELD: + feedId = xcp.text(); + break; + default: + xcp.skipChildren(); + } + } + + return new IocDao( + id, + name, + type, + value, + severity, + specVersion, + created, + modified, + description, + labels, + feedId + ); + } + + /** + * Validates required fields. + * @throws IllegalArgumentException + */ + public void validate() throws IllegalArgumentException { + if (type == null) { + throw new IllegalArgumentException(String.format("[%s] is required.", TYPE_FIELD)); + } else if (!Arrays.asList(IocType.values()).contains(type)) { + logger.debug("Unsupported IocType: {}", type); + throw new IllegalArgumentException(String.format("[%s] is not supported.", TYPE_FIELD)); + } + + if (value == null || value.isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] is required.", VALUE_FIELD)); + } + + if (feedId == null || feedId.isEmpty()) { + throw new IllegalArgumentException(String.format("[%s] is required.", FEED_ID_FIELD)); + } + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public IocType getType() { + return type; + } + + public String getValue() { + return value; + } + + public String getSeverity() { + return severity; + } + + public String getSpecVersion() { + return specVersion; + } + + public Instant getCreated() { + return created; + } + + public Instant getModified() { + return modified; + } + + public String getDescription() { + return description; + } + + public List getLabels() { + return labels; + } + + public String getFeedId() { + return feedId; + } + + public enum IocType { + DOMAIN("domain") { + @Override + public String getSystemIndexName() { + return IOC_DOMAIN_INDEX_NAME; + } + }, + HASH("hash") { // TODO placeholder + @Override + public String getSystemIndexName() { + return IOC_HASH_INDEX_NAME; + } + }, + IP("ip") { + @Override + public String getSystemIndexName() { + return IOC_IP_INDEX_NAME; + } + }; + + IocType(String type) {} + + public abstract String getSystemIndexName(); + } +} diff --git a/src/main/java/org/opensearch/securityanalytics/model/IocDto.java b/src/main/java/org/opensearch/securityanalytics/model/IocDto.java new file mode 100644 index 000000000..ca9163cf8 --- /dev/null +++ b/src/main/java/org/opensearch/securityanalytics/model/IocDto.java @@ -0,0 +1,140 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.core.xcontent.XContentParserUtils; + +import java.io.IOException; +import java.time.Instant; +import java.util.Collections; +import java.util.List; +import java.util.Locale; + +public class IocDto implements Writeable, ToXContentObject { + private static final Logger logger = LogManager.getLogger(IocDto.class); + + private String id; + private String name; + private IocDao.IocType type; + private String value; + private String severity; + private String specVersion; + private Instant created; + private Instant modified; + private String description; + private List labels; + private String feedId; + + public IocDto(IocDao iocDao) { + this.id = iocDao.getId(); + this.name = iocDao.getName(); + this.type = iocDao.getType(); + this.value = iocDao.getValue(); + this.severity = iocDao.getSeverity(); + this.specVersion = iocDao.getSpecVersion(); + this.created = iocDao.getCreated(); + this.modified = iocDao.getModified(); + this.description = iocDao.getDescription(); + this.labels = iocDao.getLabels(); + this.feedId = iocDao.getFeedId(); + } + + public IocDto(StreamInput sin) throws IOException { + this(new IocDao(sin)); + } + + public static IocDto readFrom(StreamInput sin) throws IOException { + return new IocDto(sin); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeString(name); + out.writeEnum(type); + out.writeString(value); + out.writeString(severity); + out.writeString(specVersion); + out.writeInstant(created); + out.writeInstant(modified); + out.writeString(description); + out.writeStringCollection(labels); + out.writeString(feedId); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + return builder.startObject() + .field(IocDao.ID_FIELD, id) + .field(IocDao.NAME_FIELD, name) + .field(IocDao.TYPE_FIELD, type) + .field(IocDao.VALUE_FIELD, value) + .field(IocDao.SEVERITY_FIELD, severity) + .field(IocDao.SPEC_VERSION_FIELD, specVersion) + .timeField(IocDao.CREATED_FIELD, created) + .timeField(IocDao.MODIFIED_FIELD, modified) + .field(IocDao.DESCRIPTION_FIELD, description) + .field(IocDao.LABELS_FIELD, labels) + .field(IocDao.FEED_ID_FIELD, feedId) + .endObject(); + } + + public static IocDto parse(XContentParser xcp, String id) throws IOException { + return new IocDto(IocDao.parse(xcp, id)); + } + + public String getId() { + return id; + } + + public String getName() { + return name; + } + + public IocDao.IocType getType() { + return type; + } + + public String getValue() { + return value; + } + + public String getSeverity() { + return severity; + } + + public String getSpecVersion() { + return specVersion; + } + + public Instant getCreated() { + return created; + } + + public Instant getModified() { + return modified; + } + + public String getDescription() { + return description; + } + + public List getLabels() { + return labels; + } + + public String getFeedId() { + return feedId; + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java index 03dca9281..26f3c8216 100644 --- a/src/test/java/org/opensearch/securityanalytics/TestHelpers.java +++ b/src/test/java/org/opensearch/securityanalytics/TestHelpers.java @@ -29,6 +29,8 @@ import org.opensearch.securityanalytics.model.DetectorRule; import org.opensearch.securityanalytics.model.DetectorTrigger; import org.opensearch.securityanalytics.model.IoCMatch; +import org.opensearch.securityanalytics.model.IocDao; +import org.opensearch.securityanalytics.model.IocDto; import org.opensearch.securityanalytics.model.ThreatIntelFeedData; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.rest.OpenSearchRestTestCase; @@ -36,13 +38,16 @@ import java.io.IOException; import java.time.Instant; import java.time.temporal.ChronoUnit; +import java.time.temporal.TemporalUnit; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.stream.Collectors; +import java.util.stream.IntStream; import static org.opensearch.test.OpenSearchTestCase.randomInt; @@ -2712,4 +2717,123 @@ public static NamedXContentRegistry xContentRegistry() { public static XContentBuilder builder() throws IOException { return XContentBuilder.builder(XContentType.JSON.xContent()); } + + public static IocDao randomIocDao() { + return randomIocDao( + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ); + } + + public static IocDao randomIocDao( + String id, + String name, + IocDao.IocType type, + String value, + String severity, + String specVersion, + Instant created, + Instant modified, + String description, + List labels, + String feedId + ) { + if (id == null) { + id = randomString(); + } + if (name == null) { + name = randomString(); + } + if (type == null) { + type = IocDao.IocType.values()[randomInt(IocDao.IocType.values().length - 1)]; + } + if (value == null) { + value = randomString(); + } + if (severity == null) { + severity = randomString(); + } + if (specVersion == null) { + specVersion = randomString(); + } + if (created == null) { + created = Instant.now(); + } + if (modified == null) { + modified = Instant.now().plusSeconds(3600); // 1 hour + } + if (description == null) { + description = randomString(); + } + if (labels == null) { + labels = IntStream.range(0, randomInt()) + .mapToObj(i -> randomString()) + .collect(Collectors.toList()); + } + if (feedId == null) { + feedId = randomString(); + } + return new IocDao( + id, + name, + type, + value, + severity, + specVersion, + created, + modified, + description, + labels, + feedId + ); + } + + public static IocDto randomIocDto() { + return new IocDto(randomIocDao()); + } + + public static IocDto randomIocDto( + String id, + String name, + IocDao.IocType type, + String value, + String severity, + String specVersion, + Instant created, + Instant modified, + String description, + List labels, + String feedId + ) { + return new IocDto(randomIocDao( + id, + name, + type, + value, + severity, + specVersion, + created, + modified, + description, + labels, + feedId + )); + } + + public static XContentParser getParser(String xc) throws IOException { + XContentParser parser = XContentType.JSON.xContent() + .createParser(xContentRegistry(), LoggingDeprecationHandler.INSTANCE, xc); + parser.nextToken(); + return parser; + + } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java new file mode 100644 index 000000000..4fda1a1b4 --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocDaoTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.getParser; +import static org.opensearch.securityanalytics.TestHelpers.randomIocDao; + +public class IocDaoTests extends OpenSearchTestCase { + public void testAsStream() throws IOException { + IocDao ioc = randomIocDao(); + BytesStreamOutput out = new BytesStreamOutput(); + ioc.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocDao newIoc = new IocDao(sin); + assertEqualIocDaos(ioc, newIoc); + } + + public void testParseFunction() throws IOException { + IocDao ioc = randomIocDao(); + String json = toJsonString(ioc); + IocDao newIoc = IocDao.parse(getParser(json), ioc.getId()); + assertEqualIocDaos(ioc, newIoc); + } + + private String toJsonString(IocDao ioc) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + private void assertEqualIocDaos(IocDao ioc, IocDao newIoc) { + assertEquals(ioc.getId(), newIoc.getId()); + assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getValue(), newIoc.getValue()); + assertEquals(ioc.getSeverity(), newIoc.getSeverity()); + assertEquals(ioc.getSpecVersion(), newIoc.getSpecVersion()); + assertEquals(ioc.getCreated(), newIoc.getCreated()); + assertEquals(ioc.getModified(), newIoc.getModified()); + assertEquals(ioc.getDescription(), newIoc.getDescription()); + assertEquals(ioc.getLabels(), newIoc.getLabels()); + assertEquals(ioc.getFeedId(), newIoc.getFeedId()); + } +} diff --git a/src/test/java/org/opensearch/securityanalytics/model/IocDtoTests.java b/src/test/java/org/opensearch/securityanalytics/model/IocDtoTests.java new file mode 100644 index 000000000..c1af99dfd --- /dev/null +++ b/src/test/java/org/opensearch/securityanalytics/model/IocDtoTests.java @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.securityanalytics.model; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.common.xcontent.XContentFactory; +import org.opensearch.core.common.bytes.BytesReference; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.xcontent.ToXContent; +import org.opensearch.core.xcontent.XContentBuilder; +import org.opensearch.test.OpenSearchTestCase; + +import java.io.IOException; + +import static org.opensearch.securityanalytics.TestHelpers.getParser; +import static org.opensearch.securityanalytics.TestHelpers.randomIocDto; + +public class IocDtoTests extends OpenSearchTestCase { + public void testAsStream() throws IOException { + IocDto ioc = randomIocDto(); + BytesStreamOutput out = new BytesStreamOutput(); + ioc.writeTo(out); + StreamInput sin = StreamInput.wrap(out.bytes().toBytesRef().bytes); + IocDto newIoc = new IocDto(sin); + assertEqualIocDtos(ioc, newIoc); + } + + public void testParseFunction() throws IOException { + IocDto ioc = randomIocDto(); + String json = toJsonString(ioc); + IocDto newIoc = IocDto.parse(getParser(json), ioc.getId()); + assertEqualIocDtos(ioc, newIoc); + } + + private String toJsonString(IocDto ioc) throws IOException { + XContentBuilder builder = XContentFactory.jsonBuilder(); + builder = ioc.toXContent(builder, ToXContent.EMPTY_PARAMS); + return BytesReference.bytes(builder).utf8ToString(); + } + + private void assertEqualIocDtos(IocDto ioc, IocDto newIoc) { + assertEquals(ioc.getId(), newIoc.getId()); + assertEquals(ioc.getName(), newIoc.getName()); + assertEquals(ioc.getValue(), newIoc.getValue()); + assertEquals(ioc.getSeverity(), newIoc.getSeverity()); + assertEquals(ioc.getSpecVersion(), newIoc.getSpecVersion()); + assertEquals(ioc.getCreated(), newIoc.getCreated()); + assertEquals(ioc.getModified(), newIoc.getModified()); + assertEquals(ioc.getDescription(), newIoc.getDescription()); + assertEquals(ioc.getLabels(), newIoc.getLabels()); + assertEquals(ioc.getFeedId(), newIoc.getFeedId()); + } +}