From 3852380ae942f36be3a2c1db9af93b4898646813 Mon Sep 17 00:00:00 2001 From: "opensearch-trigger-bot[bot]" <98922864+opensearch-trigger-bot[bot]@users.noreply.github.com> Date: Wed, 9 Nov 2022 10:05:51 -0800 Subject: [PATCH] add mappings and test case (#89) (#126) * dns and cloudtrail mappings Signed-off-by: Grant Haywood * add s3 mappings Signed-off-by: Grant Haywood Signed-off-by: Grant Haywood (cherry picked from commit ee7f6fb9c5dd5470ce6c06a9c797c46a0da51b87) Co-authored-by: phaseshiftg <115187865+phaseshiftg@users.noreply.github.com> --- .../OSMapping/cloudtrail/fieldmappings.yml | 15 +- .../OSMapping/cloudtrail/mappings.json | 36 +++- .../resources/OSMapping/dns/fieldmappings.yml | 9 +- .../resources/OSMapping/dns/mappings.json | 14 +- .../resources/OSMapping/s3/fieldmappings.yml | 8 +- src/main/resources/OSMapping/s3/mappings.json | 8 +- .../mapper/MapperRestApiIT.java | 194 ++++++++++++++++++ src/test/resources/cloudtrail-sample-s3.json | 35 ++++ src/test/resources/cloudtrail-sample.json | 105 ++++++++++ src/test/resources/dns-sample.json | 50 +++++ 10 files changed, 443 insertions(+), 31 deletions(-) create mode 100644 src/test/resources/cloudtrail-sample-s3.json create mode 100644 src/test/resources/cloudtrail-sample.json create mode 100644 src/test/resources/dns-sample.json diff --git a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml index 9ab79f08f..a4540f117 100644 --- a/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml +++ b/src/main/resources/OSMapping/cloudtrail/fieldmappings.yml @@ -1,7 +1,10 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - CommandLine: windows-event_data-CommandLine + eventName: aws-cloudtrail-eventType + eventSource: aws-cloudtrail-eventSource + requestParameters.arn: aws-cloudtrail-requestParameters-arn + requestParameters.attribute: aws-cloudtrail-requestParameters-attribute + requestParameters.userName: aws-cloudtrail-requestParameters-userName + requestParameters.containerDefinitions.command: aws-cloudtrail-requestParameters-container-definitions-command + userIdentity.sessionContext.sessionIssuer.type: aws-cloudtrail-userIdentity-sessionContext-session_issuer-type + userIdentity.type: aws-cloudtrail-userIdentity-type + userIdentity.arn: aws-cloudtrail-userIdentity-type \ No newline at end of file diff --git a/src/main/resources/OSMapping/cloudtrail/mappings.json b/src/main/resources/OSMapping/cloudtrail/mappings.json index ea77f2460..7a9a35c47 100644 --- a/src/main/resources/OSMapping/cloudtrail/mappings.json +++ b/src/main/resources/OSMapping/cloudtrail/mappings.json @@ -1,12 +1,40 @@ { "properties": { - "windows-event_data-CommandLine": { + "aws-cloudtrail-eventType": { "type": "alias", - "path": "CommandLine" + "path": "aws.cloudtrail.eventType" }, - "event_uid": { + "aws-cloudtrail-eventSource": { "type": "alias", - "path": "EventID" + "path": "aws.cloudtrail.eventSource" + }, + "aws-cloudtrail-requestParameters-arn": { + "type": "alias", + "path": "aws.cloudtrail.requestParameters.arn" + }, + "aws-cloudtrail-requestParameters-attribute": { + "type": "alias", + "path": "aws.cloudtrail.requestParameters.attribute" + }, + "aws-cloudtrail-requestParameters-userName": { + "type": "alias", + "path": "aws.cloudtrail.requestParameters.userName" + }, + "aws-cloudtrail-requestParameters-containerDefinitions-command": { + "type": "alias", + "path": "aws.cloudtrail.requestParameters.containerDefinitions.command" + }, + "aws-cloudtrail-userIdentity-type": { + "type": "alias", + "path": "aws.cloudtrail.userIdentity.type" + }, + "aws-cloudtrail-userIdentity-arn": { + "type": "alias", + "path": "aws.cloudtrail.userIdentity.arn" + }, + "userIdentity-sessionContext-sessionIssuer-type": { + "type": "alias", + "path": "aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type" } } } \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/fieldmappings.yml b/src/main/resources/OSMapping/dns/fieldmappings.yml index 9ab79f08f..fc85e1ee7 100644 --- a/src/main/resources/OSMapping/dns/fieldmappings.yml +++ b/src/main/resources/OSMapping/dns/fieldmappings.yml @@ -1,7 +1,4 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - CommandLine: windows-event_data-CommandLine + record_type: dns-answers-type + query: dns-question-name + parent_domain: dns-question-registered_domain \ No newline at end of file diff --git a/src/main/resources/OSMapping/dns/mappings.json b/src/main/resources/OSMapping/dns/mappings.json index ea77f2460..6f9f869ea 100644 --- a/src/main/resources/OSMapping/dns/mappings.json +++ b/src/main/resources/OSMapping/dns/mappings.json @@ -1,12 +1,16 @@ { "properties": { - "windows-event_data-CommandLine": { + "dns-answers-type": { "type": "alias", - "path": "CommandLine" + "path": "dns.answers.type" }, - "event_uid": { + "dns-question-name": { "type": "alias", - "path": "EventID" + "path": "dns.question.name" + }, + "dns-question-registered_domain": { + "type": "alias", + "path": "dns.question.registered_domain" } } -} \ No newline at end of file +} diff --git a/src/main/resources/OSMapping/s3/fieldmappings.yml b/src/main/resources/OSMapping/s3/fieldmappings.yml index 9ab79f08f..d2cbd9dc9 100644 --- a/src/main/resources/OSMapping/s3/fieldmappings.yml +++ b/src/main/resources/OSMapping/s3/fieldmappings.yml @@ -1,7 +1,3 @@ -# this file provides pre-defined mappings for Sigma fields defined for all Sigma rules under windows log group to their corresponding ECS Fields. fieldmappings: - EventID: event_uid - HiveName: unmapped.HiveName - fieldB: mappedB - fieldA1: mappedA - CommandLine: windows-event_data-CommandLine + eventName: aws-cloudtrail-eventName + eventSource: aws-cloudtrail-eventSource diff --git a/src/main/resources/OSMapping/s3/mappings.json b/src/main/resources/OSMapping/s3/mappings.json index ea77f2460..2559999ad 100644 --- a/src/main/resources/OSMapping/s3/mappings.json +++ b/src/main/resources/OSMapping/s3/mappings.json @@ -1,12 +1,12 @@ { "properties": { - "windows-event_data-CommandLine": { + "aws-cloudtrail-eventSource": { "type": "alias", - "path": "CommandLine" + "path": "aws.cloudtrail.eventSource" }, - "event_uid": { + "aws-cloudtrail-eventName": { "type": "alias", - "path": "EventID" + "path": "aws.cloudtrail.eventName" } } } \ No newline at end of file diff --git a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java index 32fd81db8..375344308 100644 --- a/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java +++ b/src/test/java/org/opensearch/securityanalytics/mapper/MapperRestApiIT.java @@ -10,17 +10,25 @@ import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.common.Strings; import org.opensearch.common.settings.Settings; +import org.opensearch.common.xcontent.DeprecationHandler; +import org.opensearch.common.xcontent.NamedXContentRegistry; +import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentParser; import org.opensearch.common.xcontent.json.JsonXContent; import org.opensearch.securityanalytics.SecurityAnalyticsClientUtils; import org.opensearch.securityanalytics.SecurityAnalyticsPlugin; import org.opensearch.securityanalytics.SecurityAnalyticsRestTestCase; +import java.io.BufferedReader; import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class MapperRestApiIT extends SecurityAnalyticsRestTestCase { @@ -323,4 +331,190 @@ private void createSampleIndex(String indexName) throws IOException { assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); } + + private final String DNS_SAMPLE = "dns-sample.json"; + private final String CLOUDTRAIL_SAMPLE = "cloudtrail-sample.json"; + private final String CLOUDTRAIL_SAMPLE_S3 = "cloudtrail-sample-s3.json"; + + + private final String DNS_MAPPINGS = "OSMapping/dns/mappings.json"; + private final String CLOUDTRAIL_MAPPINGS = "OSMapping/cloudtrail/mappings.json"; + private final String S3_MAPPINGS = "OSMapping/s3/mappings.json"; + + private final String NETWORK_MAPPINGS = "OSMapping/network/mappings.json"; + private final String LINUX_MAPPINGS = "OSMapping/linux/mappings.json"; + private final String WINDOWS_MAPPINGS = "OSMapping/windows/mappings.json"; + private final String APACHE_ACCESS_MAPPINGS = "OSMapping/apache_access/mappings.json"; + private final String AD_LDAP_MAPPINGS = "OSMapping/ad_ldap/mappings.json"; + + private String readResource(String name) throws IOException { + try (InputStream inputStream = SecurityAnalyticsPlugin.class.getClassLoader().getResourceAsStream(name)) { + if (inputStream == null) { + throw new IOException("Resource not found: " + name); + } + try (BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + } + + public void testReadResource() throws IOException { + String content = readResource(DNS_MAPPINGS); + assertTrue(content.contains("properties")); + } + + public void testCreateCloudTrailMappingS3() throws IOException { + String INDEX_NAME = "test_create_cloudtrail_s3_mapping_index"; + + createSampleIndex(INDEX_NAME); + // Sample dns document + String sampleDoc = readResource(CLOUDTRAIL_SAMPLE_S3); + // Index doc + Request indexRequest = new Request("POST", INDEX_NAME + "/_doc?refresh=wait_for"); + indexRequest.setJsonEntity(sampleDoc); + //Generate automatic mappings my inserting doc + Response response = client().performRequest(indexRequest); + //Get the mappings being tested + String indexMapping = readResource(S3_MAPPINGS); + //Parse the mappings + XContentParser parser = JsonXContent.jsonXContent + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + indexMapping); + Map mappings = (Map) parser.map().get("properties"); + GetMappingsResponse getMappingsResponse = SecurityAnalyticsClientUtils.executeGetMappingsRequest(INDEX_NAME); + + MappingsTraverser mappingsTraverser = new MappingsTraverser(getMappingsResponse.getMappings().iterator().next().value); + List flatProperties = mappingsTraverser.extractFlatNonAliasFields(); + assertTrue(flatProperties.contains("aws.cloudtrail.eventName")); + assertTrue(flatProperties.contains("aws.cloudtrail.eventSource")); + //Loop over the mappings and run update request for each one specifying the index to be updated + mappings.entrySet().forEach(entry -> { + String key = entry.getKey(); + String path = ((Map) entry.getValue()).get("path").toString(); + try { + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", INDEX_NAME, + "field", path, + "alias", key)))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + // Refresh everything + response = client().performRequest(new Request("POST", "_refresh")); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + + public void testCreateCloudTrailMapping() throws IOException { + String INDEX_NAME = "test_create_cloudtrail_mapping_index"; + + createSampleIndex(INDEX_NAME); + // Sample dns document + String sampleDoc = readResource(CLOUDTRAIL_SAMPLE); + // Index doc + Request indexRequest = new Request("POST", INDEX_NAME + "/_doc?refresh=wait_for"); + indexRequest.setJsonEntity(sampleDoc); + //Generate automatic mappings my inserting doc + Response response = client().performRequest(indexRequest); + //Get the mappings being tested + String indexMapping = readResource(CLOUDTRAIL_MAPPINGS); + //Parse the mappings + XContentParser parser = JsonXContent.jsonXContent + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + indexMapping); + Map mappings = (Map) parser.map().get("properties"); + GetMappingsResponse getMappingsResponse = SecurityAnalyticsClientUtils.executeGetMappingsRequest(INDEX_NAME); + + MappingsTraverser mappingsTraverser = new MappingsTraverser(getMappingsResponse.getMappings().iterator().next().value); + List flatProperties = mappingsTraverser.extractFlatNonAliasFields(); + assertTrue(flatProperties.contains("aws.cloudtrail.eventType")); + assertTrue(flatProperties.contains("aws.cloudtrail.eventSource")); + assertTrue(flatProperties.contains("aws.cloudtrail.requestParameters.arn")); + assertTrue(flatProperties.contains("aws.cloudtrail.requestParameters.attribute")); + assertTrue(flatProperties.contains("aws.cloudtrail.requestParameters.userName")); + assertTrue(flatProperties.contains("aws.cloudtrail.userIdentity.arn")); + assertTrue(flatProperties.contains("aws.cloudtrail.userIdentity.type")); + assertTrue(flatProperties.contains("aws.cloudtrail.userIdentity.sessionContext.sessionIssuer.type")); + //Loop over the mappings and run update request for each one specifying the index to be updated + mappings.entrySet().forEach(entry -> { + String key = entry.getKey(); + String path = ((Map) entry.getValue()).get("path").toString(); + try { + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", INDEX_NAME, + "field", path, + "alias", key)))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + // Refresh everything + response = client().performRequest(new Request("POST", "_refresh")); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + public void testCreateDNSMapping() throws IOException{ + String INDEX_NAME = "test_create_cloudtrail_mapping_index"; + + createSampleIndex(INDEX_NAME); + // Sample dns document + String dnsSampleDoc = readResource(DNS_SAMPLE); + // Index doc + Request indexRequest = new Request("POST", INDEX_NAME + "/_doc?refresh=wait_for"); + indexRequest.setJsonEntity(dnsSampleDoc); + //Generate automatic mappings my inserting doc + Response response = client().performRequest(indexRequest); + //Get the mappings being tested + String indexMapping = readResource(DNS_MAPPINGS); + //Parse the mappings + XContentParser parser = JsonXContent.jsonXContent + .createParser( + NamedXContentRegistry.EMPTY, + DeprecationHandler.THROW_UNSUPPORTED_OPERATION, + indexMapping); + Map mappings = (Map) parser.map().get("properties"); + GetMappingsResponse getMappingsResponse = SecurityAnalyticsClientUtils.executeGetMappingsRequest(INDEX_NAME); + + MappingsTraverser mappingsTraverser = new MappingsTraverser(getMappingsResponse.getMappings().iterator().next().value); + List flatProperties = mappingsTraverser.extractFlatNonAliasFields(); + assertTrue(flatProperties.contains("dns.answers.type")); + assertTrue(flatProperties.contains("dns.question.name")); + assertTrue(flatProperties.contains("dns.question.registered_domain")); + + //Loop over the mappings and run update request for each one specifying the index to be updated + mappings.entrySet().forEach(entry -> { + String key = entry.getKey(); + String path = ((Map) entry.getValue()).get("path").toString(); + try { + Request updateRequest = new Request("PUT", SecurityAnalyticsPlugin.MAPPER_BASE_URI); + updateRequest.setJsonEntity(Strings.toString(XContentFactory.jsonBuilder().map(Map.of( + "index_name", INDEX_NAME, + "field", path, + "alias", key)))); + Response apiResponse = client().performRequest(updateRequest); + assertEquals(HttpStatus.SC_OK, apiResponse.getStatusLine().getStatusCode()); + + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + + // Refresh everything + response = client().performRequest(new Request("POST", "_refresh")); + assertEquals(HttpStatus.SC_OK, response.getStatusLine().getStatusCode()); + } + } diff --git a/src/test/resources/cloudtrail-sample-s3.json b/src/test/resources/cloudtrail-sample-s3.json new file mode 100644 index 000000000..779e847a2 --- /dev/null +++ b/src/test/resources/cloudtrail-sample-s3.json @@ -0,0 +1,35 @@ +{ + "aws": { + "cloudtrail": { + "eventVersion": "1.03", + "userIdentity": { + "type": "IAMUser", + "principalId": "111122223333", + "arn": "arn:aws:iam::111122223333:user/myUserName", + "accountId": "111122223333", + "accessKeyId": "AKIAIOSFODNN7EXAMPLE", + "userName": "myUserName" + }, + "eventTime": "2019-02-01T03:18:19Z", + "eventSource": "s3.amazonaws.com", + "eventName": "ListBuckets", + "awsRegion": "us-west-2", + "sourceIPAddress": "127.0.0.1", + "userAgent": "[]", + "requestParameters": { + "host": [ + "s3.us-west-2.amazonaws.com" + ] + }, + "responseElements": null, + "additionalEventData": { + "SignatureVersion": "SigV2", + "AuthenticationMethod": "QueryString" + }, + "requestID": "47B8E8D397DCE7A6", + "eventID": "cdc4b7ed-e171-4cef-975a-ad829d4123e8", + "eventType": "AwsApiCall", + "recipientAccountId": "111122223333" + } + } +} \ No newline at end of file diff --git a/src/test/resources/cloudtrail-sample.json b/src/test/resources/cloudtrail-sample.json new file mode 100644 index 000000000..c997c53ca --- /dev/null +++ b/src/test/resources/cloudtrail-sample.json @@ -0,0 +1,105 @@ +{ + "aws": { + "cloudtrail": { + "eventVersion": "1.05", + "userIdentity": { + "type": "IAMUser", + "principalId": "AIDACKCEVSQ6C2EXAMPLE", + "arn": "arn:aws:iam::123456789012:user/test-user", + "accountId": "123456789012", + "accessKeyId": "access-key", + "userName": "test-user", + "sessionContext": { + "sessionIssuer": { + "type": "Role", + "principalId": "AIDACKCEVSQ6C2EXAMPLE", + "arn": "arn:aws:iam::123456789012:role/test-role", + "accountId": "123456789012", + "userName": "test-role" + }, + "issuer": { + "type": "" + }, + "attributes": { + "mfaAuthenticated": "false", + "creationDate": "2018-08-21T21:59:11Z" + } + }, + "invokedBy": "signin.amazonaws.com" + }, + "eventTime": "2018-08-21T22:00:05Z", + "eventSource": "es.amazonaws.com", + "eventName": "CreateDomain", + "awsRegion": "us-west-1", + "sourceIPAddress": "123.123.123.123", + "userAgent": "signin.amazonaws.com", + "requestParameters": { + "attribute": "", + "userName": "", + "arn": "", + "containerDefinitions": { + "command": "" + }, + "engineVersion": "OpenSearch_1.0", + "clusterConfig": { + "instanceType": "m4.large.search", + "instanceCount": 1 + }, + "snapshotOptions": { + "automatedSnapshotStartHour": 0 + }, + "domainName": "test-domain", + "encryptionAtRestOptions": {}, + "eBSOptions": { + "eBSEnabled": true, + "volumeSize": 10, + "volumeType": "gp2" + }, + "accessPolicies": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":[\"123456789012\"]},\"Action\":[\"es:*\"],\"Resource\":\"arn:aws:es:us-west-1:123456789012:domain/test-domain/*\"}]}", + "advancedOptions": { + "rest.action.multi.allow_explicit_index": "true" + } + }, + "responseElements": { + "domainStatus": { + "created": true, + "clusterConfig": { + "zoneAwarenessEnabled": false, + "instanceType": "m4.large.search", + "dedicatedMasterEnabled": false, + "instanceCount": 1 + }, + "cognitoOptions": { + "enabled": false + }, + "encryptionAtRestOptions": { + "enabled": false + }, + "advancedOptions": { + "rest.action.multi.allow_explicit_index": "true" + }, + "upgradeProcessing": false, + "snapshotOptions": { + "automatedSnapshotStartHour": 0 + }, + "eBSOptions": { + "eBSEnabled": true, + "volumeSize": 10, + "volumeType": "gp2" + }, + "engineVersion": "OpenSearch_1.0", + "processing": true, + "aRN": "arn:aws:es:us-west-1:123456789012:domain/test-domain", + "domainId": "123456789012/test-domain", + "deleted": false, + "domainName": "test-domain", + "accessPolicies": "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Principal\":{\"AWS\":\"arn:aws:iam::123456789012:root\"},\"Action\":\"es:*\",\"Resource\":\"arn:aws:es:us-west-1:123456789012:domain/test-domain/*\"}]}" + } + }, + "requestID": "12345678-1234-1234-1234-987654321098", + "eventID": "87654321-4321-4321-4321-987654321098", + "eventType": "AwsApiCall", + "recipientAccountId": "123456789012" + } + } +} \ No newline at end of file diff --git a/src/test/resources/dns-sample.json b/src/test/resources/dns-sample.json new file mode 100644 index 000000000..1c4df84ce --- /dev/null +++ b/src/test/resources/dns-sample.json @@ -0,0 +1,50 @@ +{ + "dns": { + "additionals_count": 0, + "answers": [ + { + "class": "IN", + "data": "192.168.73.66", + "name": "chat.testdomain.loc", + "ttl": "52", + "type": "A" + } + ], + "answers_count": 1, + "authorities_count": 0, + "flags": { + "authentic_data": false, + "authoritative": false, + "checking_disabled": false, + "recursion_available": true, + "recursion_desired": true, + "truncated_response": false + }, + "header_flags": [ + "RD", + "RA" + ], + "id": 59295, + "op_code": "QUERY", + "opt": { + "do": false, + "ext_rcode": "NOERROR", + "udp_size": 512, + "version": "0" + }, + "question": { + "class": "IN", + "etld_plus_one": "testdomain.loc", + "name": "chat.testdomain.loc", + "registered_domain": "testdomain.loc", + "subdomain": "chat", + "top_level_domain": "loc", + "type": "A" + }, + "resolved_ip": [ + "192.168.73.66" + ], + "response_code": "NOERROR", + "type": "answer" + } +} \ No newline at end of file