diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index d6671bcf36..23e76440c4 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -34,6 +34,7 @@ CHECK_PACKAGES_TESTS=( test-check-packages-with-kind test-check-packages-with-custom-agent test-check-packages-benchmarks + test-check-packages-with-logstash ) for test in ${CHECK_PACKAGES_TESTS[@]}; do echo " - label: \":go: Running integration test: ${test}\"" diff --git a/Makefile b/Makefile index ebda29d07c..8ef78ae89c 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ test-stack-command-8x: test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x -test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives +test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-logstash test-check-packages-with-kind: PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh @@ -81,6 +81,9 @@ test-check-packages-other: test-check-packages-false-positives: PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh +test-check-packages-with-logstash: + PACKAGE_TEST_TYPE=with-logstash ./scripts/test-check-packages.sh + test-check-packages-benchmarks: PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh diff --git a/cmd/stack.go b/cmd/stack.go index 95270564e3..123edc1a87 100644 --- a/cmd/stack.go +++ b/cmd/stack.go @@ -24,6 +24,7 @@ var availableServices = map[string]struct{}{ "fleet-server": {}, "kibana": {}, "package-registry": {}, + "logstash": {}, } const stackLongDescription = `Use this command to spin up a Docker-based Elastic Stack consisting of Elasticsearch, Kibana, and the Package Registry. By default the latest released version of the stack is spun up but it is possible to specify a different version, including SNAPSHOT versions by appending --version . diff --git a/docs/howto/custom_images.md b/docs/howto/custom_images.md index fef71777fa..67fb6b6096 100644 --- a/docs/howto/custom_images.md +++ b/docs/howto/custom_images.md @@ -20,6 +20,7 @@ The current images that could be overwritten are: | Elasticsearch | ELASTICSEARCH_IMAGE_REF_OVERRIDE | elasticsearch | | Kibana | KIBANA_IMAGE_REF_OVERRIDE | kibana | | Elastic Agent | ELASTIC_AGENT_IMAGE_REF_OVERRIDE | elastic-agent | +| Logstash | LOGSTASH_IMAGE_REF_OVERRIDE | logstash | For the following two examples, it will be used as example overwriting elastic-agent image. diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 15655e7021..8831f95da1 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -579,6 +579,25 @@ Example `expected_errors` file content: * observed hit count 4 did not match expected hit count 2 ``` +### System testing with logstash + +It is possible to test packages that output to Logstash which in turn publishes events to Elasticsearch. +A profile config option `stack.logstash_enabled` has been added to profile configuration. + +When this profile config is enabled +- Logstash output is added in Fleet with id `fleet-logstash-output` +- Logstash service is created in the stack which reads from `elastic-agent` input and outputs to `elasticsearch`. +- Logstash is also configured with `elastic-integration` plugin. Once configured to point to an Elasticsearch cluster, this filter will detect which ingest pipeline (if any) should be executed for each event, auto-detecting the event’s data-stream and its default pipeline. + +A sample workflow would look like: + +- You can [create](https://github.com/elastic/elastic-package#elastic-package-profiles-create) a new profile / [use existing profile](https://github.com/elastic/elastic-package#elastic-package-profiles-use) to test this. +- Navigate to `~/.elastic-package/profiles//`. +- Rename `config.yml.example` to `config.yml` [ If config is not used before ] +- Add the following line (or uncomment if present) `stack.logstash_enabled: true` +- Run `elastic-package stack up -d -v` +- Navigate to the package folder in integrations and run `elastic-package test system -v` + ## Continuous Integration `elastic-package` runs a set of system tests on some [dummy packages](https://github.com/elastic/elastic-package/tree/main/test/packages) to ensure it's functionalities work as expected. This allows to test changes affecting package testing within `elastic-package` before merging and releasing the changes. diff --git a/internal/install/application_configuration.go b/internal/install/application_configuration.go index a131ffab82..bdac93cc62 100644 --- a/internal/install/application_configuration.go +++ b/internal/install/application_configuration.go @@ -29,6 +29,7 @@ const ( elasticAgentCompleteImageName = "docker.elastic.co/elastic-agent/elastic-agent-complete" elasticsearchImageName = "docker.elastic.co/elasticsearch/elasticsearch" kibanaImageName = "docker.elastic.co/kibana/kibana" + logstashImageName = "docker.elastic.co/logstash/logstash" applicationConfigurationYmlFile = "config.yml" ) @@ -87,6 +88,7 @@ func (s stack) ImageRefOverridesForVersion(version string) ImageRefs { ElasticAgent: checkImageRefOverride("ELASTIC_AGENT_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.ElasticAgent, "")), Elasticsearch: checkImageRefOverride("ELASTICSEARCH_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Elasticsearch, "")), Kibana: checkImageRefOverride("KIBANA_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Kibana, "")), + Logstash: checkImageRefOverride("LOGSTASH_IMAGE_REF_OVERRIDE", stringOrDefault(appConfigImageRefs.Logstash, "")), } } @@ -95,6 +97,7 @@ type ImageRefs struct { ElasticAgent string `yaml:"elastic-agent"` Elasticsearch string `yaml:"elasticsearch"` Kibana string `yaml:"kibana"` + Logstash string `yaml:"logstash"` } // AsEnv method returns key=value representation of image refs. @@ -103,6 +106,7 @@ func (ir ImageRefs) AsEnv() []string { vars = append(vars, "ELASTIC_AGENT_IMAGE_REF="+ir.ElasticAgent) vars = append(vars, "ELASTICSEARCH_IMAGE_REF="+ir.Elasticsearch) vars = append(vars, "KIBANA_IMAGE_REF="+ir.Kibana) + vars = append(vars, "LOGSTASH_IMAGE_REF="+ir.Logstash) return vars } @@ -112,6 +116,7 @@ func (ac *ApplicationConfiguration) StackImageRefs(version string) ImageRefs { refs.ElasticAgent = stringOrDefault(refs.ElasticAgent, fmt.Sprintf("%s:%s", selectElasticAgentImageName(version), version)) refs.Elasticsearch = stringOrDefault(refs.Elasticsearch, fmt.Sprintf("%s:%s", elasticsearchImageName, version)) refs.Kibana = stringOrDefault(refs.Kibana, fmt.Sprintf("%s:%s", kibanaImageName, version)) + refs.Logstash = stringOrDefault(refs.Logstash, fmt.Sprintf("%s:%s", logstashImageName, version)) return refs } diff --git a/internal/kibana/policies.go b/internal/kibana/policies.go index 8ed3c37029..8e48019c02 100644 --- a/internal/kibana/policies.go +++ b/internal/kibana/policies.go @@ -21,6 +21,7 @@ type Policy struct { Revision int `json:"revision,omitempty"` MonitoringEnabled []string `json:"monitoring_enabled,omitempty"` MonitoringOutputID string `json:"monitoring_output_id,omitempty"` + DataOutputID string `json:"data_output_id,omitempty"` } // CreatePolicy persists the given Policy in Fleet. diff --git a/internal/profile/_static/config.yml.example b/internal/profile/_static/config.yml.example index e0f700f692..76a28a71bf 100644 --- a/internal/profile/_static/config.yml.example +++ b/internal/profile/_static/config.yml.example @@ -10,3 +10,7 @@ # Region where the Serverless project is going to be created # stack.serverless.region: aws-us-east-1 +## Enable logstash for testing +# Flag to enable logstash in elastic-package stack profile config +# stack.logstash_enabled: true + diff --git a/internal/profile/_testdata/config.yml b/internal/profile/_testdata/config.yml index 77df9e7837..0eb5991f74 100644 --- a/internal/profile/_testdata/config.yml +++ b/internal/profile/_testdata/config.yml @@ -1,5 +1,6 @@ # An expected setting. stack.geoip_dir: "/home/foo/Documents/ingest-geoip" +stack.logstash_enabled: true # An empty string, should exist, but return empty. other.empty: "" diff --git a/internal/profile/config_test.go b/internal/profile/config_test.go index 2d6ee947da..5f7644d3d1 100644 --- a/internal/profile/config_test.go +++ b/internal/profile/config_test.go @@ -47,6 +47,11 @@ func TestLoadProfileConfig(t *testing.T) { expected: "false", found: true, }, + { + name: "stack.logstash_enabled", + expected: "true", + found: true, + }, { name: "not.present", found: false, diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl index d1033c19cc..7b4e72c10e 100644 --- a/internal/stack/_static/docker-compose-stack.yml.tmpl +++ b/internal/stack/_static/docker-compose-stack.yml.tmpl @@ -147,3 +147,37 @@ services: depends_on: elastic-agent: condition: service_healthy + +{{ $logstash_enabled := fact "logstash_enabled" }} +{{ if eq $logstash_enabled "true" }} + logstash: + depends_on: + elasticsearch: + condition: service_healthy + kibana: + condition: service_healthy + image: ${LOGSTASH_IMAGE_REF} + healthcheck: + test: bin/logstash -t + interval: 60s + timeout: 50s + retries: 5 + command: bash -c "bin/logstash-plugin install logstash-filter-elastic_integration && logstash -f /usr/share/logstash/pipeline/logstash.conf" + volumes: + - "../certs/logstash:/usr/share/logstash/config/certs" + - "../certs/elasticsearch/cert.pem:/usr/share/logstash/config/certs/elasticsearch.pem" + - "./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro" + ports: + - "127.0.0.1:5044:5044" + environment: + - xpack.monitoring.enabled=false + - ELASTIC_USER=elastic + - ELASTIC_PASSWORD=changeme + - ELASTIC_HOSTS=https://127.0.0.1:9200 + + logstash_is_ready: + image: tianon/true + depends_on: + logstash: + condition: service_healthy +{{ end }} diff --git a/internal/stack/_static/kibana.yml.tmpl b/internal/stack/_static/kibana.yml.tmpl index 3e75831eab..739711eea8 100644 --- a/internal/stack/_static/kibana.yml.tmpl +++ b/internal/stack/_static/kibana.yml.tmpl @@ -80,4 +80,12 @@ xpack.fleet.outputs: ca_trusted_fingerprint: "${ELASTIC_PACKAGE_CA_TRUSTED_FINGERPRINT}" is_default: true is_default_monitoring: true + + {{ $logstash_enabled := fact "logstash_enabled" }} + {{ if eq $logstash_enabled "true" }} + - id: fleet-logstash-output + name: logstash-output + type: logstash + hosts: [ logstash:5044 ] + {{ end }} {{ end }} diff --git a/internal/stack/_static/logstash.conf.tmpl b/internal/stack/_static/logstash.conf.tmpl new file mode 100644 index 0000000000..88687dcfec --- /dev/null +++ b/internal/stack/_static/logstash.conf.tmpl @@ -0,0 +1,33 @@ +input { + elastic_agent { + port => 5044 + ssl_enabled => false + ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca-cert.pem"] + ssl_certificate => "/usr/share/logstash/config/certs/cert.pem" + ssl_key => "/usr/share/logstash/config/certs/key.pem" + } +} + + +filter { + elastic_integration { + remove_field => ['@version'] + hosts => ["https://elasticsearch:9200"] + username => {{ fact "username" }} + password => {{ fact "password" }} + ssl_enabled => true + ssl_verification_mode => "none" + } +} + + +output { + elasticsearch { + hosts => ["https://elasticsearch:9200"] + user => {{ fact "username" }} + password => {{ fact "password" }} + ssl_enabled => true + ssl_certificate_authorities => "/usr/share/logstash/config/certs/elasticsearch.pem" + data_stream => "true" + } +} diff --git a/internal/stack/certs.go b/internal/stack/certs.go index ac63774e6a..7416d6e072 100644 --- a/internal/stack/certs.go +++ b/internal/stack/certs.go @@ -23,6 +23,7 @@ var tlsServices = []string{ "kibana", "package-registry", "fleet-server", + "logstash", } var ( diff --git a/internal/stack/resources.go b/internal/stack/resources.go index 05701c0587..362b7356e1 100644 --- a/internal/stack/resources.go +++ b/internal/stack/resources.go @@ -32,6 +32,9 @@ const ( // KibanaConfigFile is the kibana config file. KibanaConfigFile = "kibana.yml" + // LogstashConfigFile is the logstash config file. + LogstashConfigFile = "logstash.conf" + // KibanaHealthcheckFile is the kibana healthcheck. KibanaHealthcheckFile = "kibana_healthcheck.sh" @@ -91,6 +94,10 @@ var ( Path: KibanaConfigFile, Content: staticSource.Template("_static/kibana.yml.tmpl"), }, + &resource.File{ + Path: LogstashConfigFile, + Content: staticSource.Template("_static/logstash.conf.tmpl"), + }, &resource.File{ Path: KibanaHealthcheckFile, Content: staticSource.Template("_static/kibana_healthcheck.sh.tmpl"), @@ -122,7 +129,8 @@ func applyResources(profile *profile.Profile, stackVersion string) error { "username": elasticsearchUsername, "password": elasticsearchPassword, - "geoip_dir": profile.Config("stack.geoip_dir", "./ingest-geoip"), + "geoip_dir": profile.Config("stack.geoip_dir", "./ingest-geoip"), + "logstash_enabled": profile.Config("stack.logstash_enabled", "false"), }) os.MkdirAll(stackDir, 0755) diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index 0f5f397a15..dc11de5128 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -516,11 +516,17 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // Configure package (single data stream) via Ingest Manager APIs. logger.Debug("creating test policy...") testTime := time.Now().Format("20060102T15:04:05Z") + p := kibana.Policy{ Name: fmt.Sprintf("ep-test-system-%s-%s-%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream, testTime), Description: fmt.Sprintf("test policy created by elastic-package test system for data stream %s/%s", r.options.TestFolder.Package, r.options.TestFolder.DataStream), Namespace: "ep", } + // Assign the data_output_id to the agent policy to configure the output to logstash. The value is inferred from stack/_static/kibana.yml.tmpl + if r.options.Profile.Config("stack.logstash_enabled", "false") == "true" { + p.DataOutputID = "fleet-logstash-output" + } + policy, err := r.options.KibanaClient.CreatePolicy(p) if err != nil { return result.WithError(fmt.Errorf("could not create test policy: %w", err)) diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 384fe4add3..2e013ed676 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -22,6 +22,11 @@ cleanup() { # Take down the stack elastic-package stack down -v + if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then + # Delete the logstash profile + elastic-package profiles delete logstash -v + fi + # Clean used resources for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do ( @@ -47,6 +52,18 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do done cd - +if [ "${PACKAGE_TEST_TYPE:-other}" == "with-logstash" ]; then + # Create a logstash profile and use it + elastic-package profiles create logstash -v + elastic-package profiles use logstash + + # Rename the config.yml.example to config.yml + mv ~/.elastic-package/profiles/logstash/config.yml.example ~/.elastic-package/profiles/logstash/config.yml + + # Append config to enable logstash + echo "stack.logstash_enabled: true" >> ~/.elastic-package/profiles/logstash/config.yml +fi + # Update the stack elastic-package stack update -v diff --git a/test/packages/with-logstash/ti_misp/_dev/build/build.yml b/test/packages/with-logstash/ti_misp/_dev/build/build.yml new file mode 100644 index 0000000000..2e15cac656 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: "git@v8.10.0" diff --git a/test/packages/with-logstash/ti_misp/_dev/deploy/docker/docker-compose.yml b/test/packages/with-logstash/ti_misp/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..c4312bb558 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: "2.3" +services: + misp: + image: docker.elastic.co/observability/stream:v0.6.1 + ports: + - 8080 + volumes: + - ./files:/files:ro + environment: + PORT: 8080 + command: + - http-server + - --addr=:8080 + - --config=/files/config.yml diff --git a/test/packages/with-logstash/ti_misp/_dev/deploy/docker/files/config.yml b/test/packages/with-logstash/ti_misp/_dev/deploy/docker/files/config.yml new file mode 100644 index 0000000000..2a1726b067 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/_dev/deploy/docker/files/config.yml @@ -0,0 +1,506 @@ +rules: + - path: /events/restSearch + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: |- + { + "response": [ + { + "Event": { + "Attribute": [ + { + "Galaxy": [], + "ShadowAttribute": [], + "category": "Payload delivery", + "comment": "filename content for test event 3", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "266263", + "last_seen": null, + "object_id": "0", + "object_relation": null, + "sharing_group_id": "0", + "timestamp": "1621589229", + "to_ids": false, + "type": "filename", + "uuid": "3b322e1a-1dd8-490c-ab96-12e1bc3ee6a3", + "value": "thetestfile.txt" + } + ], + "EventReport": [], + "Galaxy": [], + "Object": [ + { + "Attribute": [ + { + "Galaxy": [], + "ShadowAttribute": [], + "category": "Payload delivery", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "266265", + "last_seen": null, + "object_id": "18207", + "object_relation": "sha256", + "sharing_group_id": "0", + "timestamp": "1621589548", + "to_ids": true, + "type": "sha256", + "uuid": "657c5f2b-9d68-4ff7-a9ad-ab9e6a6c953e", + "value": "f33c27745f2bd87344be790465ef984a972fd539dc83bd4f61d4242c607ef1ee" + } + ], + "ObjectReference": [], + "comment": "File object for event 3", + "deleted": false, + "description": "File object describing a file with meta-information", + "distribution": "5", + "event_id": "3633", + "first_seen": null, + "id": "18207", + "last_seen": null, + "meta-category": "file", + "name": "file", + "sharing_group_id": "0", + "template_uuid": "688c46fb-5edb-40a3-8273-1af7923e2215", + "template_version": "22", + "timestamp": "1621589548", + "uuid": "42a88ad4-6834-46a9-a18b-aff9e078a4ea" + } + ], + "Org": { + "id": "1", + "local": true, + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "1", + "local": true, + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "RelatedEvent": [ + { + "Event": { + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "analysis": "0", + "date": "2021-05-21", + "distribution": "1", + "id": "3631", + "info": "Test event 1 just atrributes", + "org_id": "1", + "orgc_id": "1", + "published": false, + "threat_level_id": "1", + "timestamp": "1621588162", + "uuid": "8ca56ae9-3747-4172-93d2-808da1a4eaf3" + } + } + ], + "ShadowAttribute": [], + "analysis": "0", + "attribute_count": "6", + "date": "2021-05-21", + "disable_correlation": false, + "distribution": "1", + "event_creator_email": "admin@admin.test", + "extends_uuid": "", + "id": "3633", + "info": "Test event 3 objects and attributes", + "locked": false, + "org_id": "1", + "orgc_id": "1", + "proposal_email_lock": false, + "publish_timestamp": "0", + "published": false, + "sharing_group_id": "0", + "threat_level_id": "1", + "timestamp": "1621592532", + "uuid": "4edb20c7-8175-484d-bdcd-fce6872c1ef3" + } + } + ] + } + - path: /events/restSearch + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: |- + { + "response": [ + { + "Event": { + "Attribute": [ + { + "Galaxy": [], + "ShadowAttribute": [], + "category": "Network activity", + "comment": "Conext for domain type attribute event 2", + "deleted": false, + "disable_correlation": false, + "distribution": "5", + "event_id": "3632", + "first_seen": null, + "id": "266260", + "last_seen": null, + "object_id": "0", + "object_relation": null, + "sharing_group_id": "0", + "timestamp": "1621588744", + "to_ids": true, + "type": "domain", + "uuid": "a52a1b47-a580-4f33-96ba-939cf9146c9b", + "value": "baddom.madeup.local" + } + ], + "EventReport": [], + "Galaxy": [], + "Object": [], + "Org": { + "id": "1", + "local": true, + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "1", + "local": true, + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "RelatedEvent": [ + { + "Event": { + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "78acad2d-cc2d-4785-94d6-b428a0070488" + }, + "Orgc": { + "id": "2", + "name": "CIRCL", + "uuid": "55f6ea5e-2c60-40e5-964f-47a8950d210f" + }, + "analysis": "2", + "date": "2018-03-26", + "distribution": "3", + "id": "684", + "info": "OSINT - Forgot About Default Accounts? No Worries, GoScanSSH Didn’t", + "org_id": "1", + "orgc_id": "2", + "published": true, + "threat_level_id": "3", + "timestamp": "1523865236", + "uuid": "5acdb4d0-b534-4713-9612-4a1d950d210f" + } + } + ], + "ShadowAttribute": [], + "analysis": "0", + "attribute_count": "4", + "date": "2021-05-21", + "disable_correlation": false, + "distribution": "1", + "event_creator_email": "admin@admin.test", + "extends_uuid": "", + "id": "3632", + "info": "Test event 2 just more atrributes", + "locked": false, + "org_id": "1", + "orgc_id": "1", + "proposal_email_lock": false, + "publish_timestamp": "0", + "published": false, + "sharing_group_id": "0", + "threat_level_id": "2", + "timestamp": "1621588836", + "uuid": "efbca287-edb5-4ad7-b8e4-fe9da514a763" + } + }, + { + "Event": { + "id": "2", + "orgc_id": "2", + "org_id": "1", + "date": "2014-10-03", + "threat_level_id": "2", + "info": "OSINT New Indicators of Compromise for APT Group Nitro Uncovered blog post by Palo Alto Networks", + "published": true, + "uuid": "54323f2c-e50c-4268-896c-4867950d210b", + "attribute_count": "29", + "analysis": "2", + "timestamp": "1412579577", + "distribution": "3", + "proposal_email_lock": false, + "locked": false, + "publish_timestamp": "1610622316", + "sharing_group_id": "0", + "disable_correlation": false, + "extends_uuid": "", + "Org": { + "id": "1", + "name": "ORGNAME", + "uuid": "5877549f-ea76-4b91-91fb-c72ad682b4a5", + "local": true + }, + "Orgc": { + "id": "2", + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f", + "local": false + }, + "Attribute": [ + { + "id": "12394", + "type": "domain", + "category": "Network activity", + "to_ids": false, + "uuid": "572b4ab3-1af0-4d91-9cd5-07a1c0a8ab16", + "event_id": "22", + "distribution": "5", + "timestamp": "1462454963", + "comment": "", + "sharing_group_id": "0", + "deleted": false, + "disable_correlation": false, + "object_id": "0", + "object_relation": null, + "first_seen": null, + "last_seen": null, + "value": "whatsapp.com", + "Galaxy": [], + "ShadowAttribute": [] + } + ], + "ShadowAttribute": [], + "RelatedEvent": [], + "Galaxy": [], + "Object": [], + "EventReport": [], + "Tag": [ + { + "id": "1", + "name": "type:OSINT", + "colour": "#004646", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null, + "is_galaxy": false, + "is_custom_galaxy": false, + "local": 0 + }, + { + "id": "2", + "name": "tlp:green", + "colour": "#339900", + "exportable": true, + "user_id": "0", + "hide_tag": false, + "numerical_value": null, + "is_galaxy": false, + "is_custom_galaxy": false, + "local": 0 + } + ] + } + } + ] + } + - path: /events/restSearch + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"3","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: |- + { + "response": [] + } + - path: /attributes/restSearch + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"1","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: |- + { + "response": { + "Attribute": [ + { + "id": "1", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "link", + "to_ids": false, + "uuid": "542e4cbd-ee78-4a57-bfb8-1fda950d210b", + "timestamp": "1412320445", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "http://labs.opendns.com/2014/10/02/opendns-and-bash/", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "2", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "link", + "to_ids": false, + "uuid": "542e4cbe-d560-4e14-9157-1fda950d210b", + "timestamp": "1412320446", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "https://gist.github.com/andrewsmhay/de1cdc63d04c2bbf8c12", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "3", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "link", + "to_ids": false, + "uuid": "542e4cbe-12a4-4345-b0a4-1fda950d210b", + "timestamp": "1412320446", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "https://gist.githubusercontent.com/andrewsmhay/de1cdc63d04c2bbf8c12/raw/f20402cf5a0c646c63c4521f60587703fe654443/iplist", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "4", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "text", + "to_ids": false, + "uuid": "542e4ccc-b8fc-44af-959d-6ead950d210b", + "timestamp": "1412320460", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "Shellshock", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + }, + { + "id": "5", + "event_id": "1", + "object_id": "0", + "object_relation": null, + "category": "External analysis", + "type": "comment", + "to_ids": false, + "uuid": "542e4ce7-6120-41c0-8793-e90e950d210b", + "timestamp": "1412320487", + "distribution": "5", + "sharing_group_id": "0", + "comment": "", + "deleted": false, + "disable_correlation": false, + "first_seen": null, + "last_seen": null, + "value": "Data encoded by David André", + "Event": { + "org_id": "1", + "distribution": "3", + "id": "1", + "info": "OSINT ShellShock scanning IPs from OpenDNS", + "orgc_id": "2", + "uuid": "542e4c9c-cadc-4f8f-bb11-6d13950d210b" + } + } + ] + } + } + - path: /attributes/restSearch + methods: ["POST"] + request_headers: + Authorization: "test" + Content-Type: application/json + request_body: /^{"limit":"10","page":"2","returnFormat":"json","timestamp":"\d+"/ + responses: + - status_code: 200 + body: "{\n \"response\": {\n \"Attribute\": []\n }\n} " diff --git a/test/packages/with-logstash/ti_misp/changelog.yml b/test/packages/with-logstash/ti_misp/changelog.yml new file mode 100644 index 0000000000..9c46b365d8 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/changelog.yml @@ -0,0 +1,229 @@ +# newer versions go on top +- version: 1.24.0 + changes: + - description: ECS version updated to 8.10.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/7920 +- version: 1.23.0 + changes: + - description: "The format_version in the package manifest changed from 2.11.0 to 3.0.0. Removed dotted YAML keys from package manifest. Added 'owner.type: elastic' to package manifest." + type: enhancement + link: https://github.com/elastic/integrations/pull/7883 +- version: "1.22.0" + changes: + - description: Add tags.yml file so that integration's dashboards and saved searches are tagged with "Security Solution" and displayed in the Security Solution UI. + type: enhancement + link: https://github.com/elastic/integrations/pull/7789 +- version: "1.21.0" + changes: + - description: Update package-spec to 2.10.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/7570 +- version: "1.20.0" + changes: + - description: Update package to ECS 8.9.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/7111 +- version: "1.19.0" + changes: + - description: Document duration units. + type: enhancement + link: https://github.com/elastic/integrations/pull/6992 +- version: "1.18.2" + changes: + - description: Remove confusing error message tag prefix. + type: bugfix + link: https://github.com/elastic/integrations/pull/7105 +- version: "1.18.1" + changes: + - description: Remove renaming the original `message` field to `event.original` + type: bugfix + link: https://github.com/elastic/integrations/pull/6794 +- version: "1.18.0" + changes: + - description: Retain email subjects in misp.attributes. + type: enhancement + link: https://github.com/elastic/integrations/pull/6794 +- version: "1.17.0" + changes: + - description: Document valid duration units. + type: enhancement + link: https://github.com/elastic/integrations/pull/6706 +- version: "1.16.2" + changes: + - description: Fix the fingerprint processor in the Attributes Pipeline. + type: bugfix + link: https://github.com/elastic/integrations/pull/6719 +- version: "1.16.1" + changes: + - description: Keep the same timestamp for later pages in a pagination sequence. + type: bugfix + link: https://github.com/elastic/integrations/pull/6649 +- version: "1.16.0" + changes: + - description: Ensure event.kind is correctly set for pipeline errors. + type: enhancement + link: https://github.com/elastic/integrations/pull/6633 +- version: "1.15.4" + changes: + - description: Fix parsing of threat event publish_timestamp. + type: bugfix + link: https://github.com/elastic/integrations/pull/6575 +- version: "1.15.3" + changes: + - description: Fix bug where the threat_attributes data stream would not stop paginating after an empty response. + type: bugfix + link: https://github.com/elastic/integrations/pull/6512 +- version: "1.15.2" + changes: + - description: Prevent duplicate requests for the first page while paginating. + type: bugfix + link: https://github.com/elastic/integrations/pull/6495 +- version: "1.15.1" + changes: + - description: Fix timestamp format sent in query. + type: bugfix + link: https://github.com/elastic/integrations/pull/6482 +- version: "1.15.0" + changes: + - description: Update package to ECS 8.8.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/6328 +- version: "1.14.1" + changes: + - description: Fix `tags` processing that was removing original tags if misp tags were present. + type: bugfix + link: https://github.com/elastic/integrations/pull/6218 + - description: Add missing `event.original` cleanup step for `threat_attributes` data stream. + type: bugfix + link: https://github.com/elastic/integrations/pull/6218 +- version: "1.14.0" + changes: + - description: Add a new flag to enable request tracing + type: enhancement + link: https://github.com/elastic/integrations/pull/6115 +- version: "1.13.1" + changes: + - description: Harmonise object fields in data streams. + type: bugfix + link: https://github.com/elastic/integrations/pull/5917 +- version: "1.13.0" + changes: + - description: Add toggle to enable request tracing. + type: bugfix + link: https://github.com/elastic/integrations/pull/5965 +- version: "1.12.1" + changes: + - description: Harmonise distribution fields to type long. + type: bugfix + link: https://github.com/elastic/integrations/pull/5908 +- version: "1.12.0" + changes: + - description: Add Attributes datastream + type: enhancement + link: https://github.com/elastic/integrations/pull/4136 +- version: "1.11.0" + changes: + - description: Update package to ECS 8.7.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/5765 +- version: "1.10.1" + changes: + - description: Drop empty event sets. + type: bugfix + link: https://github.com/elastic/integrations/pull/5390 +- version: "1.10.0" + changes: + - description: Honor `preserve_original_event` tag. + type: enhancement + link: https://github.com/elastic/integrations/pull/5190 +- version: "1.9.0" + changes: + - description: Update package to ECS 8.6.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/4576 +- version: "1.8.0" + changes: + - description: Update package to ECS 8.5.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/4285 +- version: "1.7.1" + changes: + - description: Remove duplicate field. + type: bugfix + link: https://github.com/elastic/integrations/issues/4327 +- version: "1.7.0" + changes: + - description: Update package to ECS 8.4.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/3923 +- version: "1.6.1" + changes: + - description: Fix proxy URL documentation rendering. + type: bugfix + link: https://github.com/elastic/integrations/pull/3881 +- version: "1.6.0" + changes: + - description: Update categories to include `threat_intel`. + type: enhancement + link: https://github.com/elastic/integrations/pull/3689 +- version: "1.5.0" + changes: + - description: Update package to ECS 8.3.0. + type: enhancement + link: https://github.com/elastic/integrations/pull/3353 +- version: "1.4.1" + changes: + - description: update readme to include link to MISP documentation + type: enhancement + link: https://github.com/elastic/integrations/pull/3168 +- version: "1.4.0" + changes: + - description: Fix pagination looping forever + type: enhancement + link: https://github.com/elastic/integrations/pull/3446 +- version: "1.3.1" + changes: + - description: Update package descriptions + type: enhancement + link: https://github.com/elastic/integrations/pull/3398 +- version: "1.3.0" + changes: + - description: Update to ECS 8.2 + type: enhancement + link: https://github.com/elastic/integrations/pull/2781 +- version: "1.2.2" + changes: + - description: Add mapping for event.created + type: enhancement + link: https://github.com/elastic/integrations/pull/3042 +- version: "1.2.1" + changes: + - description: Add documentation for multi-fields + type: enhancement + link: https://github.com/elastic/integrations/pull/2916 +- version: "1.2.0" + changes: + - description: Update to ECS 8.0 + type: enhancement + link: https://github.com/elastic/integrations/pull/2448 +- version: "1.1.0" + changes: + - description: Adds dashboards and threat.feed ECS fields + type: enhancement + link: https://github.com/elastic/integrations/pull/2485 +- version: "1.0.2" + changes: + - description: Change test public IPs to the supported subset + type: bugfix + link: https://github.com/elastic/integrations/pull/2327 +- version: "1.0.1" + changes: + - description: Bump minimum version + type: enhancement + link: https://github.com/elastic/integrations/pull/2063 +- version: "1.0.0" + changes: + - description: Initial release + type: enhancement + link: https://github.com/elastic/integrations/pull/1946 diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..c689b96f1a --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/_dev/test/system/test-default-config.yml @@ -0,0 +1,13 @@ +input: httpjson +service: misp +vars: ~ +data_stream: + vars: + preserve_original_event: true + url: http://{{Hostname}}:{{Port}} + api_token: test + interval: 10m + initial_interval: 10m + enable_request_tracer: true +assert: + hit_count: 3 diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs b/test/packages/with-logstash/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..8292dd45f3 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs @@ -0,0 +1,88 @@ +config_version: "2" +interval: {{interval}} +request.method: "POST" +{{#if enable_request_tracer}} +request.tracer.filename: "../../logs/httpjson/http-request-trace-*.ndjson" +{{/if}} +{{#if url}} +request.url: {{url}}/events/restSearch +{{/if}} +{{#if ssl}} +request.ssl: {{ssl}} +{{/if}} +{{#if http_client_timeout}} +request.timeout: {{http_client_timeout}} +{{/if}} +{{#if proxy_url}} +request.proxy_url: {{proxy_url}} +{{/if}} +request.body: +{{#if filters}} + {{filters}} +{{/if}} +request.transforms: +{{#if api_token}} +- set: + target: header.Authorization + value: {{api_token}} +{{/if}} +- set: + target: body.page + value: 1 +- set: + target: body.limit + value: 10 +- set: + target: body.returnFormat + value: json +- set: + target: body.timestamp + value: '[[.cursor.timestamp.Unix]]' + default: '[[ (now (parseDuration "-{{initial_interval}}")).Unix ]]' +- set: + # Ignored by MISP, set as a workaround to make it available in response.pagination. + target: url.params.timestamp + value: '[[.body.timestamp]]' + +response.split: + target: body.response + split: + target: body.Event.Attribute + ignore_empty_value: true + keep_parent: true + split: + target: body.Event.Object + keep_parent: true + split: + target: body.Event.Object.Attribute + keep_parent: true +response.request_body_on_pagination: true +response.pagination: +- set: + target: body.page + # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1. + value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]' + fail_on_template_error: true +- set: + target: body.timestamp + value: '[[.last_response.url.params.Get "timestamp"]]' +- set: + target: url.params.timestamp + value: '[[.last_response.url.params.Get "timestamp"]]' +cursor: + timestamp: + value: '[[.last_event.Event.timestamp]]' +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 0000000000..2654b19845 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,466 @@ +--- +description: Pipeline for parsing MISP Threat Intel +processors: + #################### + # Event ECS fields # + #################### + - set: + field: ecs.version + value: '8.10.0' + - set: + field: event.kind + value: enrichment + - set: + field: event.category + value: [threat] + - set: + field: event.type + value: [indicator] + + ###################### + # General ECS fields # + ###################### + - rename: + field: message + target_field: event.original + ignore_missing: true + if: 'ctx.event?.original == null' + description: 'Renames the original `message` field to `event.original` to store a copy of the original message. The `event.original` field is not touched if the document already has one; it may happen when Logstash sends the document.' + - remove: + field: message + ignore_missing: true + if: 'ctx.event?.original != null' + description: 'The `message` field is no longer required if the document has an `event.original` field.' + - json: + field: event.original + target_field: json + - drop: + if: ctx.json?.response != null && ctx.json.response.isEmpty() + - fingerprint: + fields: + - json.Event.Attribute.uuid + - json.Event.Object.Attribute.uuid + target_field: "_id" + ignore_missing: true + - rename: + field: json.Event + target_field: misp.event + ignore_missing: true + - set: + field: threat.indicator.provider + value: misp + if: ctx.misp?.event?.Orgc?.local != 'false' + - set: + field: threat.indicator.provider + value: "{{misp.event.Orgc.name}}" + if: ctx.misp?.event?.Orgc?.local == 'false' + ignore_empty_value: true + + # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event + - remove: + field: + - misp.event.ShadowAttribute + - misp.event.RelatedEvent + - misp.event.Galaxy + - misp.event.Attribute.Galaxy + - misp.event.Attribute.ShadowAttribute + - misp.event.EventReport + - misp.event.Object.Attribute.Galaxy + - misp.event.Object.Attribute.ShadowAttribute + - misp.event.Object.ObjectReference + ignore_missing: true + - remove: + field: + - misp.event.Attribute + ignore_missing: true + if: 'ctx.misp?.event?.Attribute != null && ctx.misp.event.Attribute.size() == 0' + - remove: + field: + - misp.event.Object + ignore_missing: true + if: 'ctx.misp?.event?.Object != null && ctx.misp.event.Object.size() == 0' + - date: + field: misp.event.timestamp + tag: date_event_timestamp + formats: + - UNIX + if: ctx.misp?.event?.timestamp != null + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + if: ctx.misp?.event?.publish_timestamp != null + field: misp.event.publish_timestamp + target_field: misp.event.publish_timestamp + formats: + - UNIX + on_failure: + - append: + field: error.message + value: '{{{_ingest.on_failure_message}}}' + - rename: + field: misp.event.Attribute + target_field: misp.attribute + ignore_missing: true + - rename: + field: misp.event.Object + target_field: misp.object + ignore_missing: true + - rename: + field: misp.object.Attribute + target_field: misp.object.attribute + ignore_missing: true + - rename: + field: misp.object.meta-category + target_field: misp.object.meta_category + ignore_missing: true + - rename: + field: misp.event.Orgc + target_field: misp.orgc + ignore_missing: true + - rename: + field: misp.event.Org + target_field: misp.org + ignore_missing: true + - rename: + field: misp.event.Tag + target_field: misp.tag + ignore_missing: true + + # # Dance around issue of not being able to split the document into two. + # # Make the Object.Attribute field primary if it exists, but keep the + # # outer Attribute as context. + - rename: + field: misp.attribute + target_field: misp.context.attribute + ignore_missing: true + if: ctx.misp?.object != null + - rename: + field: misp.object.attribute + target_field: misp.attribute + ignore_missing: true + if: ctx.misp?.object != null + + - date: + field: misp.attribute.timestamp + target_field: misp.attribute.timestamp + tag: date_attribute_timestamp + formats: + - UNIX + if: ctx.misp?.attribute?.timestamp != null + on_failure: + - remove: + field: misp.attribute.timestamp + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: misp.context.attribute.timestamp + target_field: misp.context.attribute.timestamp + tag: date_context_attribute_timestamp + formats: + - UNIX + if: ctx.misp?.context?.attribute?.timestamp != null + on_failure: + - remove: + field: misp.context.attribute.timestamp + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - date: + field: misp.object.timestamp + target_field: misp.object.timestamp + tag: date_object_timestamp + formats: + - UNIX + if: ctx.misp?.object?.timestamp != null + on_failure: + - remove: + field: misp.object.timestamp + ignore_missing: true + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + + ##################### + # Threat ECS Fields # + ##################### + - set: + field: threat.feed.name + value: "MISP" + - rename: + field: misp.attribute.first_seen + target_field: threat.indicator.first_seen + ignore_missing: true + - rename: + field: misp.attribute.last_seen + target_field: threat.indicator.last_seen + ignore_missing: true + - convert: + field: misp.event.analysis + type: long + target_field: threat.indicator.scanner_stats + ignore_missing: true + - convert: + field: misp.event.threat_level_id + type: long + ignore_missing: true + + ## File/Hash indicator operations + - set: + field: threat.indicator.type + value: file + if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp.attribute.type) || ctx.misp.attribute.type.startsWith('filename'))" + - rename: + field: misp.attribute.value + target_field: "threat.indicator.file.hash.{{misp.attribute.type}}" + ignore_missing: true + if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp.attribute.type.startsWith('filename')" + - rename: + field: misp.attribute.value + target_field: threat.indicator.file.name + ignore_missing: true + if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type == 'filename'" + - grok: + field: misp.attribute.type + patterns: + - "%{WORD}\\|%{WORD:_tmp.hashtype}" + ignore_missing: true + if: ctx.misp?.attribute?.type != null && ctx.misp.attribute.type.startsWith('filename|') + - grok: + field: misp.attribute.value + patterns: + - "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}" + ignore_missing: true + if: ctx.misp?.attribute?.type != null && ctx.misp.attribute.type.startsWith('filename|') + - set: + field: threat.indicator.file.hash.{{_tmp.hashtype}} + value: "{{_tmp.hashvalue}}" + if: "ctx.misp?.attribute?.type != null && ctx.misp.attribute.type.startsWith('filename|') && ctx._tmp?.hashvalue != null && ctx._tmp.hashtype != null" + + ## URL/URI indicator operations + - set: + field: threat.indicator.type + value: url + if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp.attribute.type)" + - uri_parts: + field: misp.attribute.value + target_field: threat.indicator.url + keep_original: true + remove_if_successful: true + if: ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri' + - set: + field: threat.indicator.url.full + value: "{{{threat.indicator.url.original}}}" + ignore_empty_value: true + if: "ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri'" + + ## Regkey indicator operations + - set: + field: threat.indicator.type + value: windows-registry-key + if: "ctx.misp?.attribute?.type != null && ctx.misp.attribute.type.startsWith('regkey')" + - rename: + field: misp.attribute.value + target_field: threat.indicator.registry.key + ignore_missing: true + if: "ctx.threat?.indicator?.type == 'windows-registry-key' && ctx.misp?.attribute?.type == 'regkey'" + - grok: + field: misp.attribute.value + patterns: + - "%{DATA:threat.indicator.registry.key}\\|%{DATA:threat.indicator.registry.value}" + ignore_missing: true + if: "ctx.misp?.attribute?.type == 'regkey|value'" + + ## AS indicator operations + - set: + field: threat.indicator.type + value: autonomous-system + if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'" + - convert: + field: misp.attribute.value + type: long + target_field: threat.indicator.as.number + ignore_missing: true + if: ctx.threat?.indicator?.type == 'autonomous-system' + + ## Domain/IP/Port indicator operations + - set: + field: threat.indicator.type + value: domain-name + if: "ctx.misp?.attribute?.type != null && (ctx.misp.attribute.type == 'hostname' || ctx.misp.attribute.type.startsWith('domain'))" + - set: + field: threat.indicator.type + value: ipv4-addr + if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp.attribute.type)" + - rename: + field: misp.attribute.value + target_field: threat.indicator.url.domain + ignore_missing: true + if: "ctx.misp?.attribute?.type != null && ctx.threat?.indicator?.type == 'domain-name' && ctx.misp.attribute.type != 'domain|ip' && ctx.threat?.indicator?.url?.domain == null" + - rename: + field: misp.attribute.value + target_field: threat.indicator.ip + ignore_missing: true + if: "ctx.misp?.attribute?.type != null && ctx.threat?.indicator?.type == 'ipv4-addr' && !['domain|ip', 'ip-src|port', 'ip-dst|port'].contains(ctx.misp.attribute.type)" + - grok: + field: misp.attribute.value + patterns: + - "%{DATA:threat.indicator.url.domain}\\|%{IP:threat.indicator.ip}" + ignore_missing: true + if: ctx.misp?.attribute?.type == 'domain|ip' && ctx.threat?.indicator?.url?.domain == null + - grok: + field: misp.attribute.value + patterns: + - "%{IP:threat.indicator.ip}\\|%{NUMBER:threat.indicator.port}" + ignore_missing: true + if: "['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)" + + ## Email indicator operations + # Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp. + - set: + field: threat.indicator.type + value: email-addr + if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp.attribute.type)" + - set: + field: threat.indicator.type + value: email-message + if: "ctx.misp?.attribute?.type != null && ctx.misp.attribute.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp.attribute.type)" + - rename: + field: misp.attribute.value + target_field: threat.indicator.email.address + ignore_missing: true + if: ctx.threat?.indicator?.type == 'email-addr' + - rename: + field: misp.event.event_creator_email + target_field: user.email + ignore_missing: true + - append: + field: user.roles + value: "reporting_user" + if: ctx.user?.email != null + + ## MAC Address indicator operations + - set: + field: threat.indicator.type + value: mac-addr + if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp.attribute.type)" + - rename: + field: misp.attribute.value + target_field: threat.indicator.mac + ignore_missing: true + if: ctx.threat?.indicator?.type == 'mac-addr' + + ################### + # Tags ECS fields # + ################### + # Stripping special characters from tags + - script: + lang: painless + if: ctx.misp?.tag != null + source: | + def tags = ctx.misp.tag.stream() + .map(t -> t.name.replace('\\', '').replace('"', '')) + .collect(Collectors.toList()); + def tlpTags = tags.stream() + .filter(t -> t.startsWith('tlp:')) + .map(t -> t.replace('tlp:', '').toUpperCase()) + .collect(Collectors.toList()); + + if (ctx.tags == null) { + ctx.tags = new ArrayList(); + } + ctx.tags.addAll(tags); + ctx.threat.indicator.marking = [ 'tlp': tlpTags ]; + + ################# + # Convert types # + ################# + - convert: + field: misp.event.distribution + type: long + ignore_missing: true + - convert: + field: misp.object.distribution + type: long + ignore_missing: true + - convert: + field: misp.context.event.distribution + type: long + ignore_missing: true + - convert: + field: misp.attribute.distribution + type: long + ignore_missing: true + - convert: + field: misp.context.attribute.distribution + type: long + ignore_missing: true + - convert: + field: threat.indicator.port + type: long + ignore_missing: true + - convert: + field: misp.event.attribute_count + type: long + ignore_missing: true + + ###################### + # Cleanup processors # + ###################### + - script: + lang: painless + if: ctx.misp != null + source: | + void handleMap(Map map) { + for (def x : map.values()) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + map.values().removeIf(v -> v == null); + } + void handleList(List list) { + for (def x : list) { + if (x instanceof Map) { + handleMap(x); + } else if (x instanceof List) { + handleList(x); + } + } + } + handleMap(ctx); + # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event + - remove: + field: event.original + if: "ctx.tags == null || !(ctx.tags.contains('preserve_original_event'))" + ignore_failure: true + ignore_missing: true + - remove: + field: + - misp.attribute.value + ignore_missing: true + if: ctx.threat?.indicator?.type != null + - remove: + field: + - misp.event.Attribute.timestamp + - misp.event.timestamp + - misp.tag + - misp.org + - misp.event.analysis + - _tmp + - json + ignore_missing: true + +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: error.message + value: '{{{ _ingest.on_failure_message }}}' diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/fields/agent.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/agent.yml new file mode 100644 index 0000000000..da4e652c53 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/fields/base-fields.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/base-fields.yml new file mode 100644 index 0000000000..337375ce74 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/base-fields.yml @@ -0,0 +1,28 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset name. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: ti_misp +- name: event.dataset + type: constant_keyword + description: Event dataset + value: ti_misp.threat +- name: threat.feed.name + type: constant_keyword + description: Display friendly feed name. + value: MISP +- name: threat.feed.dashboard_id + type: constant_keyword + description: Dashboard ID used for Kibana CTI UI + value: ti_misp-56ed8040-6c7d-11ec-9bce-f7a4dc94c294 +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/fields/beats.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/beats.yml new file mode 100644 index 0000000000..cb44bb2944 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/beats.yml @@ -0,0 +1,12 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.flags + type: keyword + description: Flags for the log file. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file.path + type: keyword + description: Path to the log file. diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/fields/ecs.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/ecs.yml new file mode 100644 index 0000000000..31cdaf0274 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/ecs.yml @@ -0,0 +1,76 @@ +- external: ecs + name: ecs.version +- external: ecs + name: message +- external: ecs + name: tags +- external: ecs + name: error.message +- external: ecs + name: event.category +- external: ecs + name: event.ingested +- external: ecs + name: event.created +- external: ecs + name: event.kind +- external: ecs + name: event.type +- external: ecs + name: event.original +- external: ecs + name: user.email +- external: ecs + name: user.roles +- external: ecs + name: threat.indicator.first_seen +- external: ecs + name: threat.indicator.last_seen +- external: ecs + name: threat.indicator.scanner_stats +- external: ecs + name: threat.indicator.type +- external: ecs + name: threat.indicator.ip +- external: ecs + name: threat.indicator.url.domain +- external: ecs + name: threat.indicator.url.full +- external: ecs + name: threat.indicator.url.extension +- external: ecs + name: threat.indicator.url.original +- external: ecs + name: threat.indicator.url.path +- external: ecs + name: threat.indicator.url.port +- external: ecs + name: threat.indicator.url.scheme +- external: ecs + name: threat.indicator.url.query +- external: ecs + name: threat.indicator.email.address +- external: ecs + name: threat.indicator.provider +- external: ecs + name: threat.indicator.as.number +- external: ecs + name: threat.indicator.file.hash.md5 +- external: ecs + name: threat.indicator.file.hash.sha1 +- external: ecs + name: threat.indicator.file.hash.sha256 +- external: ecs + name: threat.indicator.marking.tlp +- external: ecs + name: threat.indicator.port +- external: ecs + name: threat.indicator.registry.key +- external: ecs + name: threat.indicator.registry.value +- external: ecs + name: threat.indicator.file.size +- external: ecs + name: threat.indicator.file.type +- external: ecs + name: threat.indicator.file.name diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/fields/fields.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/fields.yml new file mode 100644 index 0000000000..fada5dd0ac --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/fields/fields.yml @@ -0,0 +1,383 @@ +- name: misp + type: group + description: > + Fields for MISP indicators + + fields: + - name: event + type: group + description: > + Fields for MISP event + + fields: + - name: id + type: keyword + description: > + Attribute ID. + + - name: orgc_id + type: keyword + description: > + Organization Community ID of the event. + + - name: org_id + type: keyword + description: > + Organization ID of the event. + + - name: threat_level_id + type: long + description: > + Threat level from 5 to 1, where 1 is the most critical. + + - name: info + type: keyword + description: > + Additional text or information related to the event. + + - name: published + type: boolean + description: > + When the event was published. + + - name: uuid + type: keyword + description: > + The UUID of the event object. + + - name: date + type: date + description: > + The date of when the event object was created. + + - name: attribute_count + type: long + description: > + How many attributes are included in a single event object. + + - name: timestamp + type: date + description: > + The timestamp of when the event object was created. + + - name: distribution + type: long + description: > + Distribution type related to MISP. + + - name: proposal_email_lock + type: boolean + description: > + Settings configured on MISP for email lock on this event object. + + - name: locked + type: boolean + description: > + If the current MISP event object is locked or not. + + - name: publish_timestamp + type: date + description: > + At what time the event object was published + + - name: sharing_group_id + type: keyword + description: > + The ID of the grouped events or sources of the event. + + - name: disable_correlation + type: boolean + description: > + If correlation is disabled on the MISP event object. + + - name: extends_uuid + type: keyword + description: > + The UUID of the event object it might extend. + + - name: org.id + type: keyword + description: > + The organization ID related to the event object. + + - name: org.name + type: keyword + description: > + The organization name related to the event object. + + - name: org.uuid + type: keyword + description: > + The UUID of the organization related to the event object. + + - name: org.local + type: boolean + description: > + If the event object is local or from a remote source. + + - name: orgc.id + type: keyword + description: > + The Organization Community ID in which the event object was reported from. + + - name: orgc.name + type: keyword + description: > + The Organization Community name in which the event object was reported from. + + - name: orgc.uuid + type: keyword + description: > + The Organization Community UUID in which the event object was reported from. + + - name: orgc.local + type: boolean + description: > + If the Organization Community was local or synced from a remote source. + + - name: attribute.id + type: keyword + description: > + The ID of the attribute related to the event object. + + - name: attribute.type + type: keyword + description: > + The type of the attribute related to the event object. For example email, ipv4, sha1 and such. + + - name: attribute.category + type: keyword + description: > + The category of the attribute related to the event object. For example "Network Activity". + + - name: attribute.to_ids + type: boolean + description: > + If the attribute should be automatically synced with an IDS. + + - name: attribute.uuid + type: keyword + description: > + The UUID of the attribute related to the event. + + - name: attribute.event_id + type: keyword + description: > + The local event ID of the attribute related to the event. + + - name: attribute.distribution + type: long + description: > + How the attribute has been distributed, represented by integer numbers. + + - name: attribute.timestamp + type: date + description: > + The timestamp in which the attribute was attached to the event object. + + - name: attribute.comment + type: keyword + description: > + Comments made to the attribute itself. + + - name: attribute.sharing_group_id + type: keyword + description: > + The group ID of the sharing group related to the specific attribute. + + - name: attribute.deleted + type: boolean + description: > + If the attribute has been removed from the event object. + + - name: attribute.disable_correlation + type: boolean + description: > + If correlation has been enabled on the attribute related to the event object. + + - name: attribute.object_id + type: keyword + description: > + The ID of the Object in which the attribute is attached. + + - name: attribute.object_relation + type: keyword + description: > + The type of relation the attribute has with the event object itself. + + - name: attribute.value + type: keyword + description: > + The value of the attribute, depending on the type like "url, sha1, email-src". + + - name: context.attribute.id + type: keyword + description: > + The ID of the secondary attribute related to the event object. + + - name: context.attribute.type + type: keyword + description: > + The type of the secondary attribute related to the event object. For example email, ipv4, sha1 and such. + + - name: context.attribute.category + type: keyword + description: > + The category of the secondary attribute related to the event object. For example "Network Activity". + + - name: context.attribute.to_ids + type: boolean + description: > + If the secondary attribute should be automatically synced with an IDS. + + - name: context.attribute.uuid + type: keyword + description: > + The UUID of the secondary attribute related to the event. + + - name: context.attribute.event_id + type: keyword + description: > + The local event ID of the secondary attribute related to the event. + + - name: context.attribute.distribution + type: long + description: > + How the secondary attribute has been distributed, represented by integer numbers. + + - name: context.attribute.timestamp + type: date + description: > + The timestamp in which the secondary attribute was attached to the event object. + + - name: context.attribute.comment + type: keyword + description: > + Comments made to the secondary attribute itself. + + - name: context.attribute.sharing_group_id + type: keyword + description: > + The group ID of the sharing group related to the specific secondary attribute. + + - name: context.attribute.deleted + type: boolean + description: > + If the secondary attribute has been removed from the event object. + + - name: context.attribute.disable_correlation + type: boolean + description: > + If correlation has been enabled on the secondary attribute related to the event object. + + - name: context.attribute.object_id + type: keyword + description: > + The ID of the Object in which the secondary attribute is attached. + + - name: context.attribute.object_relation + type: keyword + description: > + The type of relation the secondary attribute has with the event object itself. + + - name: context.attribute.value + type: keyword + description: > + The value of the attribute, depending on the type like "url, sha1, email-src". + + - name: context.attribute.first_seen + type: keyword + description: > + The first time the indicator was seen. + + - name: context.attribute.last_seen + type: keyword + description: > + The last time the indicator was seen. + + - name: object + type: group + description: > + Fields for MISP Object + + fields: + - name: id + type: keyword + description: > + The ID of the object in which the attribute is attached. + + - name: name + type: keyword + description: > + The name of the object in which the attribute is attached. + + - name: meta_category + type: keyword + description: > + The meta-category of the object in which the attribute is attached. + + - name: description + type: keyword + description: > + The description of the object in which the attribute is attached. + + - name: template_uuid + type: keyword + description: > + The UUID of attribute object's template. + + - name: template_version + type: keyword + description: > + The version of attribute object's template. + + - name: event_id + type: keyword + description: > + The event ID of the object in which the attribute is attached. + + - name: uuid + type: keyword + description: > + The UUID of the object in which the attribute is attached. + + - name: timestamp + type: date + description: > + The timestamp when the object was created. + + - name: distribution + type: long + description: > + The distribution of the object indicating who can see the object. + + - name: sharing_group_id + type: keyword + description: > + The ID of the Sharing Group the object is shared with. + + - name: comment + type: keyword + description: > + Comments made to the object in which the attribute is attached. + + - name: deleted + type: boolean + description: > + If the object in which the attribute is attached has been removed. + + - name: first_seen + type: keyword + description: > + The first time the indicator of the object was seen. + + - name: last_seen + type: keyword + description: > + The last time the indicator of the object was seen. + + - name: attribute + type: flattened + description: > + List of attributes of the object in which the attribute is attached. + diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/manifest.yml b/test/packages/with-logstash/ti_misp/data_stream/threat/manifest.yml new file mode 100644 index 0000000000..063d60bab6 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/manifest.yml @@ -0,0 +1,112 @@ +type: logs +title: MISP +streams: + - input: httpjson + vars: + - name: url + type: text + title: MISP URL + multi: false + required: true + show_user: true + default: https://mispserver.com + description: The URL or hostname of the MISP instance. + - name: api_token + type: password + title: MISP API Token + multi: false + required: true + show_user: true + description: The API token used to access the MISP instance. + - name: initial_interval + type: text + title: Initial interval + multi: false + required: true + show_user: true + default: 120h + description: How far back to look for indicators the first time the agent is started. Supported units for this parameter are h/m/s. + - name: http_client_timeout + type: text + title: HTTP Client Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h. + multi: false + required: false + show_user: false + default: 30s + - name: filters + type: yaml + title: MISP API Filters + multi: false + required: false + show_user: false + default: | + #type: + # OR: + # - ip-src + # - ip-dst + #tags: + # NOT: + # - tlp-red + description: Filters documented at [MISP API Documentation](https://www.circl.lu/doc/misp/automation/#search) is supported. + - name: proxy_url + type: text + title: Proxy URL + multi: false + required: false + show_user: false + description: URL to proxy connections in the form of http\[s\]://:@: + - name: interval + type: text + title: Interval + description: Interval at which the logs will be pulled. Supported units for this parameter are h/m/s. + multi: false + required: true + show_user: true + default: 10m + - name: ssl + type: yaml + title: SSL + multi: false + required: false + show_user: false + default: | + #verification_mode: none + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - forwarded + - misp-threat + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: enable_request_tracer + type: bool + title: Enable request tracing + multi: false + required: false + show_user: false + description: > + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details. + + template_path: httpjson.yml.hbs + title: MISP + description: Collect indicators from the MISP API diff --git a/test/packages/with-logstash/ti_misp/data_stream/threat/sample_event.json b/test/packages/with-logstash/ti_misp/data_stream/threat/sample_event.json new file mode 100644 index 0000000000..248dcb4528 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/data_stream/threat/sample_event.json @@ -0,0 +1,106 @@ +{ + "@timestamp": "2014-10-06T07:12:57.000Z", + "agent": { + "ephemeral_id": "24754055-2625-498c-8778-8566dbc8a368", + "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.9.1" + }, + "data_stream": { + "dataset": "ti_misp.threat", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.10.0" + }, + "elastic_agent": { + "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082", + "snapshot": false, + "version": "8.9.1" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "threat" + ], + "created": "2023-08-28T15:43:07.992Z", + "dataset": "ti_misp.threat", + "ingested": "2023-08-28T15:43:09Z", + "kind": "enrichment", + "original": "{\"Event\":{\"Attribute\":{\"Galaxy\":[],\"ShadowAttribute\":[],\"category\":\"Network activity\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"22\",\"first_seen\":null,\"id\":\"12394\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1462454963\",\"to_ids\":false,\"type\":\"domain\",\"uuid\":\"572b4ab3-1af0-4d91-9cd5-07a1c0a8ab16\",\"value\":\"whatsapp.com\"},\"EventReport\":[],\"Galaxy\":[],\"Object\":[],\"Org\":{\"id\":\"1\",\"local\":true,\"name\":\"ORGNAME\",\"uuid\":\"5877549f-ea76-4b91-91fb-c72ad682b4a5\"},\"Orgc\":{\"id\":\"2\",\"local\":false,\"name\":\"CthulhuSPRL.be\",\"uuid\":\"55f6ea5f-fd34-43b8-ac1d-40cb950d210f\"},\"RelatedEvent\":[],\"ShadowAttribute\":[],\"Tag\":[{\"colour\":\"#004646\",\"exportable\":true,\"hide_tag\":false,\"id\":\"1\",\"is_custom_galaxy\":false,\"is_galaxy\":false,\"local\":0,\"name\":\"type:OSINT\",\"numerical_value\":null,\"user_id\":\"0\"},{\"colour\":\"#339900\",\"exportable\":true,\"hide_tag\":false,\"id\":\"2\",\"is_custom_galaxy\":false,\"is_galaxy\":false,\"local\":0,\"name\":\"tlp:green\",\"numerical_value\":null,\"user_id\":\"0\"}],\"analysis\":\"2\",\"attribute_count\":\"29\",\"date\":\"2014-10-03\",\"disable_correlation\":false,\"distribution\":\"3\",\"extends_uuid\":\"\",\"id\":\"2\",\"info\":\"OSINT New Indicators of Compromise for APT Group Nitro Uncovered blog post by Palo Alto Networks\",\"locked\":false,\"org_id\":\"1\",\"orgc_id\":\"2\",\"proposal_email_lock\":false,\"publish_timestamp\":\"1610622316\",\"published\":true,\"sharing_group_id\":\"0\",\"threat_level_id\":\"2\",\"timestamp\":\"1412579577\",\"uuid\":\"54323f2c-e50c-4268-896c-4867950d210b\"}}", + "type": [ + "indicator" + ] + }, + "input": { + "type": "httpjson" + }, + "misp": { + "attribute": { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": 5, + "event_id": "22", + "id": "12394", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "2016-05-05T13:29:23.000Z", + "to_ids": false, + "type": "domain", + "uuid": "572b4ab3-1af0-4d91-9cd5-07a1c0a8ab16" + }, + "event": { + "attribute_count": 29, + "date": "2014-10-03", + "disable_correlation": false, + "distribution": 3, + "extends_uuid": "", + "id": "2", + "info": "OSINT New Indicators of Compromise for APT Group Nitro Uncovered blog post by Palo Alto Networks", + "locked": false, + "org_id": "1", + "orgc_id": "2", + "proposal_email_lock": false, + "publish_timestamp": "2021-01-14T11:05:16.000Z", + "published": true, + "sharing_group_id": "0", + "threat_level_id": 2, + "uuid": "54323f2c-e50c-4268-896c-4867950d210b" + }, + "orgc": { + "id": "2", + "local": false, + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + }, + "tags": [ + "preserve_original_event", + "forwarded", + "misp-threat", + "type:OSINT", + "tlp:green" + ], + "threat": { + "feed": { + "name": "MISP" + }, + "indicator": { + "marking": { + "tlp": [ + "GREEN" + ] + }, + "provider": "misp", + "scanner_stats": 2, + "type": "domain-name", + "url": { + "domain": "whatsapp.com" + } + } + } +} \ No newline at end of file diff --git a/test/packages/with-logstash/ti_misp/docs/README.md b/test/packages/with-logstash/ti_misp/docs/README.md new file mode 100644 index 0000000000..636c58c210 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/docs/README.md @@ -0,0 +1,430 @@ +# MISP Integration + +The MISP integration uses the [REST API from the running MISP instance](https://www.circl.lu/doc/misp/automation/#automation-api) to retrieve indicators and Threat Intelligence. + +## Logs + +### Threat + +The MISP integration configuration allows to set the polling interval, how far back it +should look initially, and optionally any filters used to filter the results. + +The filters themselves are based on the [MISP API documentation](https://www.circl.lu/doc/misp/automation/#search) and should support all documented fields. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset name. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Path to the log file. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| misp.attribute.category | The category of the attribute related to the event object. For example "Network Activity". | keyword | +| misp.attribute.comment | Comments made to the attribute itself. | keyword | +| misp.attribute.deleted | If the attribute has been removed from the event object. | boolean | +| misp.attribute.disable_correlation | If correlation has been enabled on the attribute related to the event object. | boolean | +| misp.attribute.distribution | How the attribute has been distributed, represented by integer numbers. | long | +| misp.attribute.event_id | The local event ID of the attribute related to the event. | keyword | +| misp.attribute.id | The ID of the attribute related to the event object. | keyword | +| misp.attribute.object_id | The ID of the Object in which the attribute is attached. | keyword | +| misp.attribute.object_relation | The type of relation the attribute has with the event object itself. | keyword | +| misp.attribute.sharing_group_id | The group ID of the sharing group related to the specific attribute. | keyword | +| misp.attribute.timestamp | The timestamp in which the attribute was attached to the event object. | date | +| misp.attribute.to_ids | If the attribute should be automatically synced with an IDS. | boolean | +| misp.attribute.type | The type of the attribute related to the event object. For example email, ipv4, sha1 and such. | keyword | +| misp.attribute.uuid | The UUID of the attribute related to the event. | keyword | +| misp.attribute.value | The value of the attribute, depending on the type like "url, sha1, email-src". | keyword | +| misp.context.attribute.category | The category of the secondary attribute related to the event object. For example "Network Activity". | keyword | +| misp.context.attribute.comment | Comments made to the secondary attribute itself. | keyword | +| misp.context.attribute.deleted | If the secondary attribute has been removed from the event object. | boolean | +| misp.context.attribute.disable_correlation | If correlation has been enabled on the secondary attribute related to the event object. | boolean | +| misp.context.attribute.distribution | How the secondary attribute has been distributed, represented by integer numbers. | long | +| misp.context.attribute.event_id | The local event ID of the secondary attribute related to the event. | keyword | +| misp.context.attribute.first_seen | The first time the indicator was seen. | keyword | +| misp.context.attribute.id | The ID of the secondary attribute related to the event object. | keyword | +| misp.context.attribute.last_seen | The last time the indicator was seen. | keyword | +| misp.context.attribute.object_id | The ID of the Object in which the secondary attribute is attached. | keyword | +| misp.context.attribute.object_relation | The type of relation the secondary attribute has with the event object itself. | keyword | +| misp.context.attribute.sharing_group_id | The group ID of the sharing group related to the specific secondary attribute. | keyword | +| misp.context.attribute.timestamp | The timestamp in which the secondary attribute was attached to the event object. | date | +| misp.context.attribute.to_ids | If the secondary attribute should be automatically synced with an IDS. | boolean | +| misp.context.attribute.type | The type of the secondary attribute related to the event object. For example email, ipv4, sha1 and such. | keyword | +| misp.context.attribute.uuid | The UUID of the secondary attribute related to the event. | keyword | +| misp.context.attribute.value | The value of the attribute, depending on the type like "url, sha1, email-src". | keyword | +| misp.event.attribute_count | How many attributes are included in a single event object. | long | +| misp.event.date | The date of when the event object was created. | date | +| misp.event.disable_correlation | If correlation is disabled on the MISP event object. | boolean | +| misp.event.distribution | Distribution type related to MISP. | long | +| misp.event.extends_uuid | The UUID of the event object it might extend. | keyword | +| misp.event.id | Attribute ID. | keyword | +| misp.event.info | Additional text or information related to the event. | keyword | +| misp.event.locked | If the current MISP event object is locked or not. | boolean | +| misp.event.org_id | Organization ID of the event. | keyword | +| misp.event.orgc_id | Organization Community ID of the event. | keyword | +| misp.event.proposal_email_lock | Settings configured on MISP for email lock on this event object. | boolean | +| misp.event.publish_timestamp | At what time the event object was published | date | +| misp.event.published | When the event was published. | boolean | +| misp.event.sharing_group_id | The ID of the grouped events or sources of the event. | keyword | +| misp.event.threat_level_id | Threat level from 5 to 1, where 1 is the most critical. | long | +| misp.event.timestamp | The timestamp of when the event object was created. | date | +| misp.event.uuid | The UUID of the event object. | keyword | +| misp.object.attribute | List of attributes of the object in which the attribute is attached. | flattened | +| misp.object.comment | Comments made to the object in which the attribute is attached. | keyword | +| misp.object.deleted | If the object in which the attribute is attached has been removed. | boolean | +| misp.object.description | The description of the object in which the attribute is attached. | keyword | +| misp.object.distribution | The distribution of the object indicating who can see the object. | long | +| misp.object.event_id | The event ID of the object in which the attribute is attached. | keyword | +| misp.object.first_seen | The first time the indicator of the object was seen. | keyword | +| misp.object.id | The ID of the object in which the attribute is attached. | keyword | +| misp.object.last_seen | The last time the indicator of the object was seen. | keyword | +| misp.object.meta_category | The meta-category of the object in which the attribute is attached. | keyword | +| misp.object.name | The name of the object in which the attribute is attached. | keyword | +| misp.object.sharing_group_id | The ID of the Sharing Group the object is shared with. | keyword | +| misp.object.template_uuid | The UUID of attribute object's template. | keyword | +| misp.object.template_version | The version of attribute object's template. | keyword | +| misp.object.timestamp | The timestamp when the object was created. | date | +| misp.object.uuid | The UUID of the object in which the attribute is attached. | keyword | +| misp.org.id | The organization ID related to the event object. | keyword | +| misp.org.local | If the event object is local or from a remote source. | boolean | +| misp.org.name | The organization name related to the event object. | keyword | +| misp.org.uuid | The UUID of the organization related to the event object. | keyword | +| misp.orgc.id | The Organization Community ID in which the event object was reported from. | keyword | +| misp.orgc.local | If the Organization Community was local or synced from a remote source. | boolean | +| misp.orgc.name | The Organization Community name in which the event object was reported from. | keyword | +| misp.orgc.uuid | The Organization Community UUID in which the event object was reported from. | keyword | +| tags | List of keywords used to tag each event. | keyword | +| threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | +| threat.feed.name | Display friendly feed name. | constant_keyword | +| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | +| threat.indicator.file.hash.md5 | MD5 hash. | keyword | +| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | +| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | +| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | +| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | +| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | +| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | +| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | +| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | +| threat.indicator.provider | The name of the indicator's provider. | keyword | +| threat.indicator.registry.key | Hive-relative path of keys. | keyword | +| threat.indicator.registry.value | Name of the value written. | keyword | +| threat.indicator.scanner_stats | Count of AV/EDR vendors that successfully detected malicious file or URL. | long | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | +| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | +| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | +| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | +| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | +| threat.indicator.url.port | Port of the request, such as 443. | long | +| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| user.email | User email address. | keyword | +| user.roles | Array of user roles at the time of the event. | keyword | + + +An example event for `threat` looks as following: + +```json +{ + "@timestamp": "2014-10-06T07:12:57.000Z", + "agent": { + "ephemeral_id": "24754055-2625-498c-8778-8566dbc8a368", + "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.9.1" + }, + "data_stream": { + "dataset": "ti_misp.threat", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.10.0" + }, + "elastic_agent": { + "id": "5607d6f4-6e45-4c33-a087-2e07de5f0082", + "snapshot": false, + "version": "8.9.1" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "threat" + ], + "created": "2023-08-28T15:43:07.992Z", + "dataset": "ti_misp.threat", + "ingested": "2023-08-28T15:43:09Z", + "kind": "enrichment", + "original": "{\"Event\":{\"Attribute\":{\"Galaxy\":[],\"ShadowAttribute\":[],\"category\":\"Network activity\",\"comment\":\"\",\"deleted\":false,\"disable_correlation\":false,\"distribution\":\"5\",\"event_id\":\"22\",\"first_seen\":null,\"id\":\"12394\",\"last_seen\":null,\"object_id\":\"0\",\"object_relation\":null,\"sharing_group_id\":\"0\",\"timestamp\":\"1462454963\",\"to_ids\":false,\"type\":\"domain\",\"uuid\":\"572b4ab3-1af0-4d91-9cd5-07a1c0a8ab16\",\"value\":\"whatsapp.com\"},\"EventReport\":[],\"Galaxy\":[],\"Object\":[],\"Org\":{\"id\":\"1\",\"local\":true,\"name\":\"ORGNAME\",\"uuid\":\"5877549f-ea76-4b91-91fb-c72ad682b4a5\"},\"Orgc\":{\"id\":\"2\",\"local\":false,\"name\":\"CthulhuSPRL.be\",\"uuid\":\"55f6ea5f-fd34-43b8-ac1d-40cb950d210f\"},\"RelatedEvent\":[],\"ShadowAttribute\":[],\"Tag\":[{\"colour\":\"#004646\",\"exportable\":true,\"hide_tag\":false,\"id\":\"1\",\"is_custom_galaxy\":false,\"is_galaxy\":false,\"local\":0,\"name\":\"type:OSINT\",\"numerical_value\":null,\"user_id\":\"0\"},{\"colour\":\"#339900\",\"exportable\":true,\"hide_tag\":false,\"id\":\"2\",\"is_custom_galaxy\":false,\"is_galaxy\":false,\"local\":0,\"name\":\"tlp:green\",\"numerical_value\":null,\"user_id\":\"0\"}],\"analysis\":\"2\",\"attribute_count\":\"29\",\"date\":\"2014-10-03\",\"disable_correlation\":false,\"distribution\":\"3\",\"extends_uuid\":\"\",\"id\":\"2\",\"info\":\"OSINT New Indicators of Compromise for APT Group Nitro Uncovered blog post by Palo Alto Networks\",\"locked\":false,\"org_id\":\"1\",\"orgc_id\":\"2\",\"proposal_email_lock\":false,\"publish_timestamp\":\"1610622316\",\"published\":true,\"sharing_group_id\":\"0\",\"threat_level_id\":\"2\",\"timestamp\":\"1412579577\",\"uuid\":\"54323f2c-e50c-4268-896c-4867950d210b\"}}", + "type": [ + "indicator" + ] + }, + "input": { + "type": "httpjson" + }, + "misp": { + "attribute": { + "category": "Network activity", + "comment": "", + "deleted": false, + "disable_correlation": false, + "distribution": 5, + "event_id": "22", + "id": "12394", + "object_id": "0", + "sharing_group_id": "0", + "timestamp": "2016-05-05T13:29:23.000Z", + "to_ids": false, + "type": "domain", + "uuid": "572b4ab3-1af0-4d91-9cd5-07a1c0a8ab16" + }, + "event": { + "attribute_count": 29, + "date": "2014-10-03", + "disable_correlation": false, + "distribution": 3, + "extends_uuid": "", + "id": "2", + "info": "OSINT New Indicators of Compromise for APT Group Nitro Uncovered blog post by Palo Alto Networks", + "locked": false, + "org_id": "1", + "orgc_id": "2", + "proposal_email_lock": false, + "publish_timestamp": "2021-01-14T11:05:16.000Z", + "published": true, + "sharing_group_id": "0", + "threat_level_id": 2, + "uuid": "54323f2c-e50c-4268-896c-4867950d210b" + }, + "orgc": { + "id": "2", + "local": false, + "name": "CthulhuSPRL.be", + "uuid": "55f6ea5f-fd34-43b8-ac1d-40cb950d210f" + } + }, + "tags": [ + "preserve_original_event", + "forwarded", + "misp-threat", + "type:OSINT", + "tlp:green" + ], + "threat": { + "feed": { + "name": "MISP" + }, + "indicator": { + "marking": { + "tlp": [ + "GREEN" + ] + }, + "provider": "misp", + "scanner_stats": 2, + "type": "domain-name", + "url": { + "domain": "whatsapp.com" + } + } + } +} +``` + +### Threat Attributes + +The MISP integration configuration allows to set the polling interval, how far back it should look initially, and optionally any filters used to filter the results. +This data stream uses the `/attributes/restSearch` API endpoint which returns more granular information regarding MISP attributes and additional information. + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| cloud.account.id | The cloud account or organization id used to identify different entities in a multi-tenant environment. Examples: AWS account id, Google Cloud ORG Id, or other unique identifier. | keyword | +| cloud.availability_zone | Availability zone in which this host is running. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| cloud.instance.id | Instance ID of the host machine. | keyword | +| cloud.instance.name | Instance name of the host machine. | keyword | +| cloud.machine.type | Machine type of the host machine. | keyword | +| cloud.project.id | Name of the project in Google Cloud. | keyword | +| cloud.provider | Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. | keyword | +| cloud.region | Region in which this host is running. | keyword | +| container.id | Unique container id. | keyword | +| container.image.name | Name of the image the container was built on. | keyword | +| container.labels | Image labels. | object | +| container.name | Container name. | keyword | +| data_stream.dataset | Data stream dataset name. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | +| error.message | Error message. | match_only_text | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | +| event.created | `event.created` contains the date/time when the event was first read by an agent, or by your pipeline. This field is distinct from `@timestamp` in that `@timestamp` typically contain the time extracted from the original event. In most situations, these two timestamps will be slightly different. The difference can be used to calculate the delay between your source generating an event, and the time when your agent first processed it. This can be used to monitor your agent's or pipeline's ability to keep up with your event source. In case the two timestamps are identical, `@timestamp` should be used. | date | +| event.dataset | Event dataset | constant_keyword | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data is coming in at a regular interval or not. | keyword | +| event.module | Event module | constant_keyword | +| event.original | Raw text message of entire event. Used to demonstrate log integrity or where the full log message (before splitting it up in multiple parts) may be required, e.g. for reindex. This field is not indexed and doc_values are disabled. It cannot be searched, but it can be retrieved from `_source`. If users wish to override this and index this field, please see `Field data types` in the `Elasticsearch Reference`. | keyword | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | +| host.architecture | Operating system architecture. | keyword | +| host.containerized | If the host is a container. | boolean | +| host.domain | Name of the domain of which the host is a member. For example, on Windows this could be the host's Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host's LDAP provider. | keyword | +| host.hostname | Hostname of the host. It normally contains what the `hostname` command returns on the host machine. | keyword | +| host.id | Unique host id. As hostname is not always unique, use values that are meaningful in your environment. Example: The current usage of `beat.name`. | keyword | +| host.ip | Host ip addresses. | ip | +| host.mac | Host mac addresses. | keyword | +| host.name | Name of the host. It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use. | keyword | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| host.os.family | OS family (such as redhat, debian, freebsd, windows). | keyword | +| host.os.kernel | Operating system kernel version as a raw string. | keyword | +| host.os.name | Operating system name, without the version. | keyword | +| host.os.name.text | Multi-field of `host.os.name`. | text | +| host.os.platform | Operating system platform (such centos, ubuntu, windows). | keyword | +| host.os.version | Operating system version as a raw string. | keyword | +| host.type | Type of host. For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment. | keyword | +| input.type | Type of Filebeat input. | keyword | +| log.file.path | Path to the log file. | keyword | +| log.flags | Flags for the log file. | keyword | +| log.offset | Offset of the entry in the log file. | long | +| message | For log events the message field contains the log message, optimized for viewing in a log viewer. For structured logs without an original message field, other fields can be concatenated to form a human-readable summary of the event. If multiple messages exist, they can be combined into one message. | match_only_text | +| misp.attribute.category | The category of the attribute. For example "Network Activity". | keyword | +| misp.attribute.comment | Comments made to the attribute itself. | keyword | +| misp.attribute.data | The data of the attribute | keyword | +| misp.attribute.decay_score | Group of fields describing decay score of the attribute | flattened | +| misp.attribute.deleted | If the attribute has been removed. | boolean | +| misp.attribute.disable_correlation | If correlation has been enabled on the attribute. | boolean | +| misp.attribute.distribution | How the attribute has been distributed, represented by integer numbers. | long | +| misp.attribute.event_id | The local event ID of the attribute. | keyword | +| misp.attribute.event_uuid | The local event UUID of the attribute. | keyword | +| misp.attribute.id | The ID of the attribute. | keyword | +| misp.attribute.object_id | The ID of the Object in which the attribute is attached. | keyword | +| misp.attribute.object_relation | The type of relation the attribute has with the attribute object itself. | keyword | +| misp.attribute.sharing_group_id | The group ID of the sharing group related to the specific attribute. | keyword | +| misp.attribute.to_ids | If the attribute should be automatically synced with an IDS. | boolean | +| misp.attribute.type | The type of the attribute. For example email, ipv4, sha1 and such. | keyword | +| misp.attribute.uuid | The UUID of the attribute. | keyword | +| misp.attribute.value | The value of the attribute, depending on the type like "url, sha1, email-src". | keyword | +| misp.event.attribute_count | How many attributes are included in a single event object. | long | +| misp.event.date | The date of when the event object was created. | date | +| misp.event.disable_correlation | If correlation is disabled on the MISP event object. | boolean | +| misp.event.distribution | Distribution type related to MISP. | long | +| misp.event.extends_uuid | The UUID of the event object it might extend. | keyword | +| misp.event.id | The local event ID of the attribute related to the event. | keyword | +| misp.event.info | Additional text or information related to the event. | keyword | +| misp.event.locked | If the current MISP event object is locked or not. | boolean | +| misp.event.org_id | Organization ID of the event. | keyword | +| misp.event.orgc_id | Organization Community ID of the event. | keyword | +| misp.event.proposal_email_lock | Settings configured on MISP for email lock on this event object. | boolean | +| misp.event.publish_timestamp | At what time the event object was published | date | +| misp.event.published | When the event was published. | boolean | +| misp.event.sharing_group_id | The ID of the grouped events or sources of the event. | keyword | +| misp.event.sighting_timestamp | At what time the event object was sighted | date | +| misp.event.threat_level_id | Threat level from 5 to 1, where 1 is the most critical. | long | +| misp.event.timestamp | The timestamp of when the event object was created. | date | +| misp.event.uuid | The UUID of the event object. | keyword | +| misp.object.attribute | List of attributes of the object in which the attribute is attached. | flattened | +| misp.object.comment | Comments made to the object in which the attribute is attached. | keyword | +| misp.object.deleted | If the object in which the attribute is attached has been removed. | boolean | +| misp.object.description | The description of the object in which the attribute is attached. | keyword | +| misp.object.distribution | The distribution of the object indicating who can see the object. | long | +| misp.object.event_id | The event ID of the object in which the attribute is attached. | keyword | +| misp.object.first_seen | The first time the indicator of the object was seen. | keyword | +| misp.object.id | The ID of the object in which the attribute is attached. | keyword | +| misp.object.last_seen | The last time the indicator of the object was seen. | keyword | +| misp.object.meta_category | The meta-category of the object in which the attribute is attached. | keyword | +| misp.object.name | The name of the object in which the attribute is attached. | keyword | +| misp.object.sharing_group_id | The ID of the Sharing Group the object is shared with. | keyword | +| misp.object.template_uuid | The UUID of attribute object's template. | keyword | +| misp.object.template_version | The version of attribute object's template. | keyword | +| misp.object.timestamp | The timestamp when the object was created. | date | +| misp.object.uuid | The UUID of the object in which the attribute is attached. | keyword | +| organization.id | Unique identifier for the organization. | keyword | +| tags | List of keywords used to tag each event. | keyword | +| threat.feed.dashboard_id | Dashboard ID used for Kibana CTI UI | constant_keyword | +| threat.feed.name | Display friendly feed name | constant_keyword | +| threat.indicator.as.number | Unique number allocated to the autonomous system. The autonomous system number (ASN) uniquely identifies each network on the Internet. | long | +| threat.indicator.email.address | Identifies a threat indicator as an email address (irrespective of direction). | keyword | +| threat.indicator.file.hash.md5 | MD5 hash. | keyword | +| threat.indicator.file.hash.sha1 | SHA1 hash. | keyword | +| threat.indicator.file.hash.sha256 | SHA256 hash. | keyword | +| threat.indicator.file.name | Name of the file including the extension, without the directory. | keyword | +| threat.indicator.file.size | File size in bytes. Only relevant when `file.type` is "file". | long | +| threat.indicator.file.type | File type (file, dir, or symlink). | keyword | +| threat.indicator.first_seen | The date and time when intelligence source first reported sighting this indicator. | date | +| threat.indicator.ip | Identifies a threat indicator as an IP address (irrespective of direction). | ip | +| threat.indicator.last_seen | The date and time when intelligence source last reported sighting this indicator. | date | +| threat.indicator.marking.tlp | Traffic Light Protocol sharing markings. | keyword | +| threat.indicator.port | Identifies a threat indicator as a port number (irrespective of direction). | long | +| threat.indicator.provider | The name of the indicator's provider. | keyword | +| threat.indicator.registry.key | Hive-relative path of keys. | keyword | +| threat.indicator.registry.value | Name of the value written. | keyword | +| threat.indicator.scanner_stats | Count of AV/EDR vendors that successfully detected malicious file or URL. | long | +| threat.indicator.type | Type of indicator as represented by Cyber Observable in STIX 2.0. | keyword | +| threat.indicator.url.domain | Domain of the url, such as "www.elastic.co". In some cases a URL may refer to an IP and/or port directly, without a domain name. In this case, the IP address would go to the `domain` field. If the URL contains a literal IPv6 address enclosed by `[` and `]` (IETF RFC 2732), the `[` and `]` characters should also be captured in the `domain` field. | keyword | +| threat.indicator.url.extension | The field contains the file extension from the original request url, excluding the leading dot. The file extension is only set if it exists, as not every url has a file extension. The leading period must not be included. For example, the value must be "png", not ".png". Note that when the file name has multiple extensions (example.tar.gz), only the last one should be captured ("gz", not "tar.gz"). | keyword | +| threat.indicator.url.full | If full URLs are important to your use case, they should be stored in `url.full`, whether this field is reconstructed or present in the event source. | wildcard | +| threat.indicator.url.full.text | Multi-field of `threat.indicator.url.full`. | match_only_text | +| threat.indicator.url.original | Unmodified original url as seen in the event source. Note that in network monitoring, the observed URL may be a full URL, whereas in access logs, the URL is often just represented as a path. This field is meant to represent the URL as it was observed, complete or not. | wildcard | +| threat.indicator.url.original.text | Multi-field of `threat.indicator.url.original`. | match_only_text | +| threat.indicator.url.path | Path of the request, such as "/search". | wildcard | +| threat.indicator.url.port | Port of the request, such as 443. | long | +| threat.indicator.url.query | The query field describes the query string of the request, such as "q=elasticsearch". The `?` is excluded from the query string. If a URL contains no `?`, there is no query field. If there is a `?` but no query, the query field exists with an empty string. The `exists` query can be used to differentiate between the two cases. | keyword | +| threat.indicator.url.scheme | Scheme of the request, such as "https". Note: The `:` is not part of the scheme. | keyword | +| user.email | User email address. | keyword | +| user.roles | Array of user roles at the time of the event. | keyword | + + diff --git a/test/packages/with-logstash/ti_misp/img/misp.svg b/test/packages/with-logstash/ti_misp/img/misp.svg new file mode 100644 index 0000000000..076530aa25 --- /dev/null +++ b/test/packages/with-logstash/ti_misp/img/misp.svg @@ -0,0 +1,158 @@ + + + + diff --git a/test/packages/with-logstash/ti_misp/manifest.yml b/test/packages/with-logstash/ti_misp/manifest.yml new file mode 100644 index 0000000000..9c687d53ca --- /dev/null +++ b/test/packages/with-logstash/ti_misp/manifest.yml @@ -0,0 +1,26 @@ +name: ti_misp +title: MISP +version: "1.24.0" +description: Ingest threat intelligence indicators from MISP platform with Elastic Agent. +type: integration +format_version: "3.0.0" +categories: ["security", "threat_intel"] +conditions: + kibana: + version: ^8.7.1 +icons: + - src: /img/misp.svg + title: MISP + size: 216x216 + type: image/svg+xml +policy_templates: + - name: ti_misp + title: MISP + description: Ingest threat intelligence indicators from MISP platform with Elastic Agent. + inputs: + - type: httpjson + title: "Ingest threat intelligence indicators from MISP platform with Elastic Agent." + description: "Ingest threat intelligence indicators from MISP platform with Elastic Agent." +owner: + github: elastic/security-external-integrations + type: elastic