From 32ff8aa553df8a320f9f7c188727c021a8cb31a2 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Fri, 8 Sep 2023 10:45:54 +0200 Subject: [PATCH 1/2] Fix tests around pipelines with custom dataset The dataset value in the manifest.yml file can be overwritten to allow dataset names which are not `{package}.{dirName}`. One example of this can be found here: https://github.com/elastic/integrations/pull/7670 This is already in use today and is used by some packages. Fleet installs all the assets as expected. When working on https://github.com/elastic/integrations/pull/7670, CI is failing. The reason is that the pipeline check did not take the custom dataset config into account. The reason why this didn't show up earlier is because the check for the template already has a similar condition in place. Likely all previous packages that used this feature did only have templates and no pipelines. --- internal/packages/assets.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/internal/packages/assets.go b/internal/packages/assets.go index d05c3943fe..2ed2ec6ad6 100644 --- a/internal/packages/assets.go +++ b/internal/packages/assets.go @@ -135,9 +135,14 @@ func loadElasticsearchAssets(pkgRootPath string) ([]Asset, error) { continue // ingest pipeline is not defined } + // If no dataset value is set in the manifest, it falls back to {packageName}.{dirName} + if dsManifest.Dataset == "" { + dsManifest.Dataset = fmt.Sprintf("%s.%s", pkgManifest.Name, dsManifest.Name) + } + ingestPipelineName := dsManifest.GetPipelineNameOrDefault() if ingestPipelineName == defaultPipelineName { - ingestPipelineName = fmt.Sprintf("%s-%s.%s-%s", dsManifest.Type, pkgManifest.Name, dsManifest.Name, pkgManifest.Version) + ingestPipelineName = fmt.Sprintf("%s-%s-%s", dsManifest.Type, dsManifest.Dataset, pkgManifest.Version) } asset = Asset{ ID: ingestPipelineName, From e6c7d4f5b7a675a232ca532aab718100bd22f584 Mon Sep 17 00:00:00 2001 From: Nicolas Ruflin Date: Mon, 11 Sep 2023 14:14:20 +0200 Subject: [PATCH 2/2] update awsfirehose package dataset to test this change --- test/packages/parallel/awsfirehose/data_stream/log/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/packages/parallel/awsfirehose/data_stream/log/manifest.yml b/test/packages/parallel/awsfirehose/data_stream/log/manifest.yml index 7bf7532a5c..b9128b556a 100644 --- a/test/packages/parallel/awsfirehose/data_stream/log/manifest.yml +++ b/test/packages/parallel/awsfirehose/data_stream/log/manifest.yml @@ -1,6 +1,6 @@ title: Logs from Amazon Kinesis Data Firehose type: logs -dataset: awsfirehose.log +dataset: awsfirehose # Ensures agents have permissions to write data to `logs-*-*` elasticsearch: dynamic_dataset: true