[system tests] Use datastream dataset if it exists

internal/testrunner/runners/system/runner.go

@@ -533,7 +533,7 @@ func createPackageDatastream(
 			Enabled: true,
 			DataStream: kibana.DataStream{
 				Type:    ds.Type,
-				Dataset: fmt.Sprintf("%s.%s", pkg.Name, ds.Name),
+				Dataset: getDataStreamDataset(pkg, ds),
 			},
 		},
 	}
@@ -594,6 +594,13 @@ func getDataStreamIndex(inputName string, ds packages.DataStreamManifest) int {
 	return 0
 }
 
+func getDataStreamDataset(pkg packages.PackageManifest, ds packages.DataStreamManifest) string {
+	if len(ds.Dataset) > 0 {
+		return ds.Dataset
+	}
+	return fmt.Sprintf("%s.%s", pkg.Name, ds.Name)
+}
+
 func deleteDataStreamDocs(api *elasticsearch.API, dataStream string) error {
 	body := strings.NewReader(`{ "query": { "match_all": {} } }`)
 	_, err := api.DeleteByQuery([]string{dataStream}, body)

test/packages/other/with_dataset/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,7 @@
+version: '2.3'
+services:
+  with_dataset:
+    image: "alpine:3.16"
+    command: ["sh", "-c", "while true; do echo '{\"message\": \"hello\"}' >> ./logs/with_dataset.log; sleep 1; done"]
+    volumes:
+      - ${SERVICE_LOGS_DIR}:/logs

test/packages/other/with_dataset/changelog.yml

@@ -0,0 +1,5 @@
+- version: "0.0.1"
+  changes:
+    - description: initial release
+      type: enhancement
+      link: https://github.com/elastic/elastic-package/pull/979

test/packages/other/with_dataset/data_stream/first/_dev/test/system/test-default-config.yml

@@ -0,0 +1,6 @@
+vars: ~
+input: logfile
+data_stream:
+  vars:
+    paths:
+      - "{{SERVICE_LOGS_DIR}}/with_dataset.log"

test/packages/other/with_dataset/data_stream/first/agent/stream/stream.yml.hbs

@@ -0,0 +1,11 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+
+processors:
+- add_fields:
+    target: ''
+    fields:
+        ecs.version: 1.6.0

test/packages/other/with_dataset/data_stream/first/fields/base-fields.yml

@@ -0,0 +1,22 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.
+- name: input.type
+  type: keyword
+- name: log.file.path
+  type: keyword
+- name: log.offset
+  type: long
+- name: ecs.version
+  type: keyword
+- name: message
+  type: match_only_text

test/packages/other/with_dataset/data_stream/first/manifest.yml

@@ -0,0 +1,14 @@
+title: Service logs
+type: logs
+dataset: with_dataset.overwritten_dataset.foo
+streams:
+  - input: logfile
+    title: Sample logs
+    description: Collect sample logs
+    vars:
+      - name: paths
+        title: Paths
+        type: text
+        required: true
+        multi: true
+        show_user: true

test/packages/other/with_dataset/data_stream/first/sample_event.json

@@ -0,0 +1,60 @@
+{
+    "@timestamp": "2022-09-20T17:53:26.894Z",
+    "agent": {
+        "ephemeral_id": "78997008-6321-4097-aeb4-9680e1e9eedf",
+        "id": "f1020af0-6e41-466e-b8d3-8e6d98624fd5",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "8.4.1"
+    },
+    "data_stream": {
+        "dataset": "with_dataset.overwritten_dataset.foo",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "1.6.0"
+    },
+    "elastic_agent": {
+        "id": "f1020af0-6e41-466e-b8d3-8e6d98624fd5",
+        "snapshot": false,
+        "version": "8.4.1"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "dataset": "with_dataset.overwritten_dataset.foo",
+        "ingested": "2022-09-20T17:53:30Z"
+    },
+    "host": {
+        "architecture": "x86_64",
+        "containerized": true,
+        "hostname": "docker-fleet-agent",
+        "id": "51511c1493f34922b559a964798246ec",
+        "ip": [
+            "192.168.32.7"
+        ],
+        "mac": [
+            "02:42:c0:a8:20:07"
+        ],
+        "name": "docker-fleet-agent",
+        "os": {
+            "codename": "focal",
+            "family": "debian",
+            "kernel": "5.10.47-linuxkit",
+            "name": "Ubuntu",
+            "platform": "ubuntu",
+            "type": "linux",
+            "version": "20.04.4 LTS (Focal Fossa)"
+        }
+    },
+    "input": {
+        "type": "log"
+    },
+    "log": {
+        "file": {
+            "path": "/tmp/service_logs/with_dataset.log"
+        },
+        "offset": 0
+    },
+    "message": "{\"message\": \"hello\"}"
+}

test/packages/other/with_dataset/docs/README.md

@@ -0,0 +1,2 @@
+# Test integration
+This package contains a data stream that overwrites the dataset used to name its assets.

test/packages/other/with_dataset/manifest.yml

@@ -0,0 +1,22 @@
+format_version: 1.0.0
+name: with_dataset
+title: With dataset test
+version: 0.0.1
+description: Package that defines system test for data streams with an overwriting dataset
+categories:
+  - custom
+release: experimental
+license: basic
+type: integration
+conditions:
+  kibana.version: '^8.0.0'
+policy_templates:
+  - name: sample
+    title: Sample logs
+    description: Collect sample logs
+    inputs:
+      - type: logfile
+        title: Collect sample logs from instances
+        description: Collecting sample logs
+owner:
+  github: elastic/integrations