diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index fdfee970da..bcb2a01b2d 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -533,7 +533,7 @@ func createPackageDatastream( Enabled: true, DataStream: kibana.DataStream{ Type: ds.Type, - Dataset: fmt.Sprintf("%s.%s", pkg.Name, ds.Name), + Dataset: getDataStreamDataset(pkg, ds), }, }, } @@ -594,6 +594,13 @@ func getDataStreamIndex(inputName string, ds packages.DataStreamManifest) int { return 0 } +func getDataStreamDataset(pkg packages.PackageManifest, ds packages.DataStreamManifest) string { + if len(ds.Dataset) > 0 { + return ds.Dataset + } + return fmt.Sprintf("%s.%s", pkg.Name, ds.Name) +} + func deleteDataStreamDocs(api *elasticsearch.API, dataStream string) error { body := strings.NewReader(`{ "query": { "match_all": {} } }`) _, err := api.DeleteByQuery([]string{dataStream}, body) diff --git a/test/packages/other/with_dataset/_dev/deploy/docker/docker-compose.yml b/test/packages/other/with_dataset/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..98e9a0c4b5 --- /dev/null +++ b/test/packages/other/with_dataset/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,7 @@ +version: '2.3' +services: + with_dataset: + image: "alpine:3.16" + command: ["sh", "-c", "while true; do echo '{\"message\": \"hello\"}' >> ./logs/with_dataset.log; sleep 1; done"] + volumes: + - ${SERVICE_LOGS_DIR}:/logs diff --git a/test/packages/other/with_dataset/changelog.yml b/test/packages/other/with_dataset/changelog.yml new file mode 100644 index 0000000000..46f8424c53 --- /dev/null +++ b/test/packages/other/with_dataset/changelog.yml @@ -0,0 +1,5 @@ +- version: "0.0.1" + changes: + - description: initial release + type: enhancement + link: https://github.com/elastic/elastic-package/pull/979 diff --git a/test/packages/other/with_dataset/data_stream/first/_dev/test/system/test-default-config.yml b/test/packages/other/with_dataset/data_stream/first/_dev/test/system/test-default-config.yml new file mode 100644 index 0000000000..1aa4b69041 --- /dev/null +++ b/test/packages/other/with_dataset/data_stream/first/_dev/test/system/test-default-config.yml @@ -0,0 +1,6 @@ +vars: ~ +input: logfile +data_stream: + vars: + paths: + - "{{SERVICE_LOGS_DIR}}/with_dataset.log" diff --git a/test/packages/other/with_dataset/data_stream/first/agent/stream/stream.yml.hbs b/test/packages/other/with_dataset/data_stream/first/agent/stream/stream.yml.hbs new file mode 100644 index 0000000000..40d041d660 --- /dev/null +++ b/test/packages/other/with_dataset/data_stream/first/agent/stream/stream.yml.hbs @@ -0,0 +1,11 @@ +paths: +{{#each paths as |path i|}} + - {{path}} +{{/each}} +exclude_files: [".gz$"] + +processors: +- add_fields: + target: '' + fields: + ecs.version: 1.6.0 diff --git a/test/packages/other/with_dataset/data_stream/first/fields/base-fields.yml b/test/packages/other/with_dataset/data_stream/first/fields/base-fields.yml new file mode 100644 index 0000000000..e8d301a72d --- /dev/null +++ b/test/packages/other/with_dataset/data_stream/first/fields/base-fields.yml @@ -0,0 +1,22 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: input.type + type: keyword +- name: log.file.path + type: keyword +- name: log.offset + type: long +- name: ecs.version + type: keyword +- name: message + type: match_only_text diff --git a/test/packages/other/with_dataset/data_stream/first/manifest.yml b/test/packages/other/with_dataset/data_stream/first/manifest.yml new file mode 100644 index 0000000000..5d9d482299 --- /dev/null +++ b/test/packages/other/with_dataset/data_stream/first/manifest.yml @@ -0,0 +1,14 @@ +title: Service logs +type: logs +dataset: with_dataset.overwritten_dataset.foo +streams: + - input: logfile + title: Sample logs + description: Collect sample logs + vars: + - name: paths + title: Paths + type: text + required: true + multi: true + show_user: true diff --git a/test/packages/other/with_dataset/data_stream/first/sample_event.json b/test/packages/other/with_dataset/data_stream/first/sample_event.json new file mode 100644 index 0000000000..01179c71e4 --- /dev/null +++ b/test/packages/other/with_dataset/data_stream/first/sample_event.json @@ -0,0 +1,60 @@ +{ + "@timestamp": "2022-09-20T17:53:26.894Z", + "agent": { + "ephemeral_id": "78997008-6321-4097-aeb4-9680e1e9eedf", + "id": "f1020af0-6e41-466e-b8d3-8e6d98624fd5", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.4.1" + }, + "data_stream": { + "dataset": "with_dataset.overwritten_dataset.foo", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "1.6.0" + }, + "elastic_agent": { + "id": "f1020af0-6e41-466e-b8d3-8e6d98624fd5", + "snapshot": false, + "version": "8.4.1" + }, + "event": { + "agent_id_status": "verified", + "dataset": "with_dataset.overwritten_dataset.foo", + "ingested": "2022-09-20T17:53:30Z" + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "id": "51511c1493f34922b559a964798246ec", + "ip": [ + "192.168.32.7" + ], + "mac": [ + "02:42:c0:a8:20:07" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.10.47-linuxkit", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "input": { + "type": "log" + }, + "log": { + "file": { + "path": "/tmp/service_logs/with_dataset.log" + }, + "offset": 0 + }, + "message": "{\"message\": \"hello\"}" +} \ No newline at end of file diff --git a/test/packages/other/with_dataset/docs/README.md b/test/packages/other/with_dataset/docs/README.md new file mode 100644 index 0000000000..da169937e7 --- /dev/null +++ b/test/packages/other/with_dataset/docs/README.md @@ -0,0 +1,2 @@ +# Test integration +This package contains a data stream that overwrites the dataset used to name its assets. diff --git a/test/packages/other/with_dataset/manifest.yml b/test/packages/other/with_dataset/manifest.yml new file mode 100644 index 0000000000..533b07e060 --- /dev/null +++ b/test/packages/other/with_dataset/manifest.yml @@ -0,0 +1,22 @@ +format_version: 1.0.0 +name: with_dataset +title: With dataset test +version: 0.0.1 +description: Package that defines system test for data streams with an overwriting dataset +categories: + - custom +release: experimental +license: basic +type: integration +conditions: + kibana.version: '^8.0.0' +policy_templates: + - name: sample + title: Sample logs + description: Collect sample logs + inputs: + - type: logfile + title: Collect sample logs from instances + description: Collecting sample logs +owner: + github: elastic/integrations