From 89b334e1c1415b952643994bd64b2df10a5ec6fc Mon Sep 17 00:00:00 2001 From: Erik Baranowski <39704712+erikbaranowski@users.noreply.github.com> Date: Tue, 5 Sep 2023 07:58:04 -0400 Subject: [PATCH] =?UTF-8?q?refactor=20promtail=20converter=20sd=20configs?= =?UTF-8?q?=20so=20they=20reuse=20prometheus=20conve=E2=80=A6=20(#5069)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * refactor promtail converter sd configs so they reuse prometheus converter functionality Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * cleanup words Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * unexport a bunch of stuff that isn't needed outside the prometheusconvert package Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * don't do SD targets in backwards order during converter Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * get rid of static SD boilerplate --------- Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> Co-authored-by: Piotr Gwizdala <17101802+thampiotr@users.noreply.github.com> --- converter/internal/prometheusconvert/azure.go | 6 +- .../internal/prometheusconvert/consul.go | 4 +- .../prometheusconvert/digitalocean.go | 6 +- .../internal/prometheusconvert/docker.go | 4 +- converter/internal/prometheusconvert/ec2.go | 6 +- converter/internal/prometheusconvert/file.go | 4 +- converter/internal/prometheusconvert/gce.go | 6 +- .../internal/prometheusconvert/kubernetes.go | 4 +- .../prometheusconvert/prometheus_blocks.go | 6 +- .../prometheusconvert/prometheusconvert.go | 12 +- .../internal/prometheusconvert/scrape.go | 6 +- .../testdata/discovery.river | 2 +- .../prometheusconvert/testdata/scrape.river | 2 +- .../internal/prometheusconvert/validate.go | 61 +++++---- .../internal/build/azure_sd.go | 23 ---- .../internal/build/consul_agent.go | 13 -- .../internal/build/consul_sd.go | 23 ---- .../internal/build/digitalocean_sd.go | 23 ---- .../internal/build/docker_sd.go | 23 ---- .../promtailconvert/internal/build/ec2_sd.go | 23 ---- .../promtailconvert/internal/build/file_sd.go | 18 --- .../promtailconvert/internal/build/gce_sd.go | 23 ---- .../internal/build/kubernetes_sd.go | 24 ---- .../internal/build/scrape_builder.go | 21 --- .../internal/build/service_discovery.go | 121 ++++++++++++++++++ .../internal/build/static_sd.go | 33 ----- .../promtailconvert/promtailconvert.go | 12 +- .../testdata/unsupported.diags | 12 +- .../staticconvert/testdata/prom_scrape.river | 4 +- .../testdata/promtail_prom.river | 2 +- 30 files changed, 201 insertions(+), 326 deletions(-) delete mode 100644 converter/internal/promtailconvert/internal/build/azure_sd.go delete mode 100644 converter/internal/promtailconvert/internal/build/consul_agent.go delete mode 100644 converter/internal/promtailconvert/internal/build/consul_sd.go delete mode 100644 converter/internal/promtailconvert/internal/build/digitalocean_sd.go delete mode 100644 converter/internal/promtailconvert/internal/build/docker_sd.go delete mode 100644 converter/internal/promtailconvert/internal/build/ec2_sd.go delete mode 100644 converter/internal/promtailconvert/internal/build/file_sd.go delete mode 100644 converter/internal/promtailconvert/internal/build/gce_sd.go delete mode 100644 converter/internal/promtailconvert/internal/build/kubernetes_sd.go create mode 100644 converter/internal/promtailconvert/internal/build/service_discovery.go delete mode 100644 converter/internal/promtailconvert/internal/build/static_sd.go diff --git a/converter/internal/prometheusconvert/azure.go b/converter/internal/prometheusconvert/azure.go index 665e401ba34d..17c41e4a042d 100644 --- a/converter/internal/prometheusconvert/azure.go +++ b/converter/internal/prometheusconvert/azure.go @@ -13,14 +13,14 @@ import ( ) func appendDiscoveryAzure(pb *prometheusBlocks, label string, sdConfig *prom_azure.SDConfig) discovery.Exports { - discoveryAzureArgs := ToDiscoveryAzure(sdConfig) + discoveryAzureArgs := toDiscoveryAzure(sdConfig) name := []string{"discovery", "azure"} block := common.NewBlockWithOverride(name, label, discoveryAzureArgs) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) return NewDiscoveryExports("discovery.azure." + label + ".targets") } -func ToDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments { +func toDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments { if sdConfig == nil { return nil } @@ -40,7 +40,7 @@ func ToDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments { } } -func ValidateDiscoveryAzure(sdConfig *prom_azure.SDConfig) diag.Diagnostics { +func validateDiscoveryAzure(sdConfig *prom_azure.SDConfig) diag.Diagnostics { return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig) } diff --git a/converter/internal/prometheusconvert/consul.go b/converter/internal/prometheusconvert/consul.go index 81971f172163..dad35e109978 100644 --- a/converter/internal/prometheusconvert/consul.go +++ b/converter/internal/prometheusconvert/consul.go @@ -12,7 +12,7 @@ import ( ) func appendDiscoveryConsul(pb *prometheusBlocks, label string, sdConfig *prom_consul.SDConfig) discovery.Exports { - discoveryConsulArgs := ToDiscoveryConsul(sdConfig) + discoveryConsulArgs := toDiscoveryConsul(sdConfig) name := []string{"discovery", "consul"} block := common.NewBlockWithOverride(name, label, discoveryConsulArgs) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) @@ -23,7 +23,7 @@ func validateDiscoveryConsul(sdConfig *prom_consul.SDConfig) diag.Diagnostics { return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig) } -func ToDiscoveryConsul(sdConfig *prom_consul.SDConfig) *consul.Arguments { +func toDiscoveryConsul(sdConfig *prom_consul.SDConfig) *consul.Arguments { if sdConfig == nil { return nil } diff --git a/converter/internal/prometheusconvert/digitalocean.go b/converter/internal/prometheusconvert/digitalocean.go index 7f031e460128..c34e90ef6a70 100644 --- a/converter/internal/prometheusconvert/digitalocean.go +++ b/converter/internal/prometheusconvert/digitalocean.go @@ -15,14 +15,14 @@ import ( ) func appendDiscoveryDigitalOcean(pb *prometheusBlocks, label string, sdConfig *prom_digitalocean.SDConfig) discovery.Exports { - discoveryDigitalOceanArgs := ToDiscoveryDigitalOcean(sdConfig) + discoveryDigitalOceanArgs := toDiscoveryDigitalOcean(sdConfig) name := []string{"discovery", "digitalocean"} block := common.NewBlockWithOverride(name, label, discoveryDigitalOceanArgs) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) return NewDiscoveryExports("discovery.digitalocean." + label + ".targets") } -func ValidateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Diagnostics { +func validateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Diagnostics { var diags diag.Diagnostics if sdConfig.HTTPClientConfig.BasicAuth != nil { @@ -46,7 +46,7 @@ func ValidateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Di return diags } -func ToDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) *digitalocean.Arguments { +func toDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) *digitalocean.Arguments { if sdConfig == nil { return nil } diff --git a/converter/internal/prometheusconvert/docker.go b/converter/internal/prometheusconvert/docker.go index c1445653e5c5..ec58f0b2b9e2 100644 --- a/converter/internal/prometheusconvert/docker.go +++ b/converter/internal/prometheusconvert/docker.go @@ -11,7 +11,7 @@ import ( ) func appendDiscoveryDocker(pb *prometheusBlocks, label string, sdConfig *prom_docker.DockerSDConfig) discovery.Exports { - discoveryDockerArgs := ToDiscoveryDocker(sdConfig) + discoveryDockerArgs := toDiscoveryDocker(sdConfig) name := []string{"discovery", "docker"} block := common.NewBlockWithOverride(name, label, discoveryDockerArgs) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) @@ -22,7 +22,7 @@ func validateDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) diag.Diagnost return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig) } -func ToDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) *docker.Arguments { +func toDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) *docker.Arguments { if sdConfig == nil { return nil } diff --git a/converter/internal/prometheusconvert/ec2.go b/converter/internal/prometheusconvert/ec2.go index 30f85d039dd1..f1641a9fb4ff 100644 --- a/converter/internal/prometheusconvert/ec2.go +++ b/converter/internal/prometheusconvert/ec2.go @@ -14,14 +14,14 @@ import ( ) func appendDiscoveryEC2(pb *prometheusBlocks, label string, sdConfig *prom_aws.EC2SDConfig) discovery.Exports { - discoveryec2Args := ToDiscoveryEC2(sdConfig) + discoveryec2Args := toDiscoveryEC2(sdConfig) name := []string{"discovery", "ec2"} block := common.NewBlockWithOverride(name, label, discoveryec2Args) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) return NewDiscoveryExports("discovery.ec2." + label + ".targets") } -func ValidateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics { +func validateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics { var diags diag.Diagnostics if sdConfig.HTTPClientConfig.BasicAuth != nil { @@ -64,7 +64,7 @@ func ValidateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics { return diags } -func ToDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) *aws.EC2Arguments { +func toDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) *aws.EC2Arguments { if sdConfig == nil { return nil } diff --git a/converter/internal/prometheusconvert/file.go b/converter/internal/prometheusconvert/file.go index dc417daa4292..4c5283e60f03 100644 --- a/converter/internal/prometheusconvert/file.go +++ b/converter/internal/prometheusconvert/file.go @@ -11,7 +11,7 @@ import ( ) func appendDiscoveryFile(pb *prometheusBlocks, label string, sdConfig *prom_file.SDConfig) discovery.Exports { - discoveryFileArgs := ToDiscoveryFile(sdConfig) + discoveryFileArgs := toDiscoveryFile(sdConfig) name := []string{"discovery", "file"} block := common.NewBlockWithOverride(name, label, discoveryFileArgs) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) @@ -22,7 +22,7 @@ func validateDiscoveryFile(sdConfig *prom_file.SDConfig) diag.Diagnostics { return make(diag.Diagnostics, 0) } -func ToDiscoveryFile(sdConfig *prom_file.SDConfig) *file.Arguments { +func toDiscoveryFile(sdConfig *prom_file.SDConfig) *file.Arguments { if sdConfig == nil { return nil } diff --git a/converter/internal/prometheusconvert/gce.go b/converter/internal/prometheusconvert/gce.go index 16b8acca5b74..f7c19a81e57f 100644 --- a/converter/internal/prometheusconvert/gce.go +++ b/converter/internal/prometheusconvert/gce.go @@ -11,18 +11,18 @@ import ( ) func appendDiscoveryGCE(pb *prometheusBlocks, label string, sdConfig *prom_gce.SDConfig) discovery.Exports { - discoveryGCEArgs := ToDiscoveryGCE(sdConfig) + discoveryGCEArgs := toDiscoveryGCE(sdConfig) name := []string{"discovery", "gce"} block := common.NewBlockWithOverride(name, label, discoveryGCEArgs) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) return NewDiscoveryExports("discovery.gce." + label + ".targets") } -func ValidateDiscoveryGCE(sdConfig *prom_gce.SDConfig) diag.Diagnostics { +func validateDiscoveryGCE(sdConfig *prom_gce.SDConfig) diag.Diagnostics { return make(diag.Diagnostics, 0) } -func ToDiscoveryGCE(sdConfig *prom_gce.SDConfig) *gce.Arguments { +func toDiscoveryGCE(sdConfig *prom_gce.SDConfig) *gce.Arguments { if sdConfig == nil { return nil } diff --git a/converter/internal/prometheusconvert/kubernetes.go b/converter/internal/prometheusconvert/kubernetes.go index eda906adbedc..c84dac76bc55 100644 --- a/converter/internal/prometheusconvert/kubernetes.go +++ b/converter/internal/prometheusconvert/kubernetes.go @@ -10,7 +10,7 @@ import ( ) func appendDiscoveryKubernetes(pb *prometheusBlocks, label string, sdConfig *prom_kubernetes.SDConfig) discovery.Exports { - discoveryKubernetesArgs := ToDiscoveryKubernetes(sdConfig) + discoveryKubernetesArgs := toDiscoveryKubernetes(sdConfig) name := []string{"discovery", "kubernetes"} block := common.NewBlockWithOverride(name, label, discoveryKubernetesArgs) pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", "")) @@ -21,7 +21,7 @@ func validateDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) diag.Diagno return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig) } -func ToDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) *kubernetes.Arguments { +func toDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) *kubernetes.Arguments { if sdConfig == nil { return nil } diff --git a/converter/internal/prometheusconvert/prometheus_blocks.go b/converter/internal/prometheusconvert/prometheus_blocks.go index 1425248fe12d..cf2b5388a28d 100644 --- a/converter/internal/prometheusconvert/prometheus_blocks.go +++ b/converter/internal/prometheusconvert/prometheus_blocks.go @@ -19,7 +19,7 @@ type prometheusBlocks struct { prometheusRemoteWriteBlocks []prometheusBlock } -func newPrometheusBlocks() *prometheusBlocks { +func NewPrometheusBlocks() *prometheusBlocks { return &prometheusBlocks{ discoveryBlocks: []prometheusBlock{}, discoveryRelabelBlocks: []prometheusBlock{}, @@ -29,7 +29,7 @@ func newPrometheusBlocks() *prometheusBlocks { } } -// appendToFile attaches prometheus blocks in a specific order. +// AppendToFile attaches prometheus blocks in a specific order. // // Order of blocks: // 1. Discovery component(s) @@ -37,7 +37,7 @@ func newPrometheusBlocks() *prometheusBlocks { // 3. Prometheus scrape component(s) // 4. Prometheus relabel component(s) (if any) // 5. Prometheus remote_write -func (pb *prometheusBlocks) appendToFile(f *builder.File) { +func (pb *prometheusBlocks) AppendToFile(f *builder.File) { for _, promBlock := range pb.discoveryBlocks { f.Body().AppendBlock(promBlock.block) } diff --git a/converter/internal/prometheusconvert/prometheusconvert.go b/converter/internal/prometheusconvert/prometheusconvert.go index 2ed715dd2b18..79e5051c71c1 100644 --- a/converter/internal/prometheusconvert/prometheusconvert.go +++ b/converter/internal/prometheusconvert/prometheusconvert.go @@ -69,7 +69,7 @@ func AppendAll(f *builder.File, promConfig *prom_config.Config) diag.Diagnostics // pipeline. Additional options can be provided overriding the job name, extra // scrape targets, and predefined remote write exports. func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToCompLabelsFunc func(string) string, extraScrapeTargets []discovery.Target, remoteWriteExports *remotewrite.Exports) diag.Diagnostics { - pb := newPrometheusBlocks() + pb := NewPrometheusBlocks() if remoteWriteExports == nil { labelPrefix := "" @@ -92,7 +92,7 @@ func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToC scrapeForwardTo = []storage.Appendable{promMetricsRelabelExports.Receiver} } - scrapeTargets := appendServiceDiscoveryConfigs(pb, scrapeConfig.ServiceDiscoveryConfigs, label) + scrapeTargets := AppendServiceDiscoveryConfigs(pb, scrapeConfig.ServiceDiscoveryConfigs, label) scrapeTargets = append(scrapeTargets, extraScrapeTargets...) promDiscoveryRelabelExports := appendDiscoveryRelabel(pb, scrapeConfig.RelabelConfigs, scrapeTargets, label) @@ -106,15 +106,15 @@ func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToC diags := validate(promConfig) diags.AddAll(pb.getScrapeInfo()) - pb.appendToFile(f) + pb.AppendToFile(f) return diags } -// appendServiceDiscoveryConfigs will loop through the service discovery +// AppendServiceDiscoveryConfigs will loop through the service discovery // configs and append them to the file. This returns the scrape targets // and discovery targets as a result. -func appendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig prom_discover.Configs, label string) []discovery.Target { +func AppendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig prom_discover.Configs, label string) []discovery.Target { var targets []discovery.Target labelCounts := make(map[string]int) for _, serviceDiscoveryConfig := range serviceDiscoveryConfig { @@ -154,7 +154,7 @@ func appendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig exports = appendDiscoveryLightsail(pb, common.LabelWithIndex(labelCounts["lightsail"]-1, label), sdc) } - targets = append(exports.Targets, targets...) + targets = append(targets, exports.Targets...) } return targets diff --git a/converter/internal/prometheusconvert/scrape.go b/converter/internal/prometheusconvert/scrape.go index 3888985f4697..5daa01ca60f8 100644 --- a/converter/internal/prometheusconvert/scrape.go +++ b/converter/internal/prometheusconvert/scrape.go @@ -5,6 +5,8 @@ import ( "strings" "time" + "golang.org/x/exp/maps" + "github.com/grafana/agent/component/discovery" "github.com/grafana/agent/component/prometheus/scrape" "github.com/grafana/agent/converter/diag" @@ -68,7 +70,9 @@ func getScrapeTargets(staticConfig prom_discovery.StaticConfig) []discovery.Targ for _, labelSet := range target.Targets { for labelName, labelValue := range labelSet { targetMap[string(labelName)] = string(labelValue) - targets = append(targets, targetMap) + newMap := map[string]string{} + maps.Copy(newMap, targetMap) + targets = append(targets, newMap) } } } diff --git a/converter/internal/prometheusconvert/testdata/discovery.river b/converter/internal/prometheusconvert/testdata/discovery.river index 2ad92149e165..4ff3cc509ce3 100644 --- a/converter/internal/prometheusconvert/testdata/discovery.river +++ b/converter/internal/prometheusconvert/testdata/discovery.river @@ -32,8 +32,8 @@ discovery.azure "prometheus1_2" { discovery.relabel "prometheus1" { targets = concat( - discovery.azure.prometheus1_2.targets, discovery.azure.prometheus1.targets, + discovery.azure.prometheus1_2.targets, [{ __address__ = "localhost:9090", }], diff --git a/converter/internal/prometheusconvert/testdata/scrape.river b/converter/internal/prometheusconvert/testdata/scrape.river index 5473dba7fc46..621a84bd1c30 100644 --- a/converter/internal/prometheusconvert/testdata/scrape.river +++ b/converter/internal/prometheusconvert/testdata/scrape.river @@ -1,7 +1,7 @@ prometheus.scrape "prometheus1" { targets = concat( [{ - __address__ = "localhost:9091", + __address__ = "localhost:9090", app = "foo", }], [{ diff --git a/converter/internal/prometheusconvert/validate.go b/converter/internal/prometheusconvert/validate.go index 349630b13395..a5715dea0719 100644 --- a/converter/internal/prometheusconvert/validate.go +++ b/converter/internal/prometheusconvert/validate.go @@ -69,34 +69,41 @@ func validateScrapeConfigs(scrapeConfigs []*prom_config.ScrapeConfig) diag.Diagn for _, scrapeConfig := range scrapeConfigs { diags.AddAll(validatePrometheusScrape(scrapeConfig)) + diags.AddAll(ValidateServiceDiscoveryConfigs(scrapeConfig.ServiceDiscoveryConfigs)) + } + + return diags +} + +func ValidateServiceDiscoveryConfigs(serviceDiscoveryConfigs prom_discover.Configs) diag.Diagnostics { + var diags diag.Diagnostics - for _, serviceDiscoveryConfig := range scrapeConfig.ServiceDiscoveryConfigs { - switch sdc := serviceDiscoveryConfig.(type) { - case prom_discover.StaticConfig: - diags.AddAll(validateScrapeTargets(sdc)) - case *prom_azure.SDConfig: - diags.AddAll(ValidateDiscoveryAzure(sdc)) - case *prom_consul.SDConfig: - diags.AddAll(validateDiscoveryConsul(sdc)) - case *prom_digitalocean.SDConfig: - diags.AddAll(ValidateDiscoveryDigitalOcean(sdc)) - case *prom_dns.SDConfig: - diags.AddAll(validateDiscoveryDns(sdc)) - case *prom_docker.DockerSDConfig: - diags.AddAll(validateDiscoveryDocker(sdc)) - case *prom_aws.EC2SDConfig: - diags.AddAll(ValidateDiscoveryEC2(sdc)) - case *prom_file.SDConfig: - diags.AddAll(validateDiscoveryFile(sdc)) - case *prom_gce.SDConfig: - diags.AddAll(ValidateDiscoveryGCE(sdc)) - case *prom_kubernetes.SDConfig: - diags.AddAll(validateDiscoveryKubernetes(sdc)) - case *prom_aws.LightsailSDConfig: - diags.AddAll(validateDiscoveryLightsail(sdc)) - default: - diags.Add(diag.SeverityLevelError, fmt.Sprintf("unsupported service discovery %s was provided", serviceDiscoveryConfig.Name())) - } + for _, serviceDiscoveryConfig := range serviceDiscoveryConfigs { + switch sdc := serviceDiscoveryConfig.(type) { + case prom_discover.StaticConfig: + diags.AddAll(validateScrapeTargets(sdc)) + case *prom_azure.SDConfig: + diags.AddAll(validateDiscoveryAzure(sdc)) + case *prom_consul.SDConfig: + diags.AddAll(validateDiscoveryConsul(sdc)) + case *prom_digitalocean.SDConfig: + diags.AddAll(validateDiscoveryDigitalOcean(sdc)) + case *prom_dns.SDConfig: + diags.AddAll(validateDiscoveryDns(sdc)) + case *prom_docker.DockerSDConfig: + diags.AddAll(validateDiscoveryDocker(sdc)) + case *prom_aws.EC2SDConfig: + diags.AddAll(validateDiscoveryEC2(sdc)) + case *prom_file.SDConfig: + diags.AddAll(validateDiscoveryFile(sdc)) + case *prom_gce.SDConfig: + diags.AddAll(validateDiscoveryGCE(sdc)) + case *prom_kubernetes.SDConfig: + diags.AddAll(validateDiscoveryKubernetes(sdc)) + case *prom_aws.LightsailSDConfig: + diags.AddAll(validateDiscoveryLightsail(sdc)) + default: + diags.Add(diag.SeverityLevelError, fmt.Sprintf("unsupported service discovery %s was provided", serviceDiscoveryConfig.Name())) } } diff --git a/converter/internal/promtailconvert/internal/build/azure_sd.go b/converter/internal/promtailconvert/internal/build/azure_sd.go deleted file mode 100644 index 445a7b9cd09e..000000000000 --- a/converter/internal/promtailconvert/internal/build/azure_sd.go +++ /dev/null @@ -1,23 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendAzureSDs() { - if len(s.cfg.ServiceDiscoveryConfig.AzureSDConfigs) == 0 { - return - } - for i, sd := range s.cfg.ServiceDiscoveryConfig.AzureSDConfigs { - s.diags.AddAll(prometheusconvert.ValidateDiscoveryAzure(sd)) - args := prometheusconvert.ToDiscoveryAzure(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride( - []string{"discovery", "azure"}, - compLabel, - args, - )) - s.allTargetsExps = append(s.allTargetsExps, "discovery.azure."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/consul_agent.go b/converter/internal/promtailconvert/internal/build/consul_agent.go deleted file mode 100644 index e4210062f211..000000000000 --- a/converter/internal/promtailconvert/internal/build/consul_agent.go +++ /dev/null @@ -1,13 +0,0 @@ -package build - -import "github.com/grafana/agent/converter/diag" - -func (s *ScrapeConfigBuilder) AppendConsulAgentSDs() { - // TODO: implement this - if s.cfg.ServiceDiscoveryConfig.ConsulAgentSDConfigs != nil { - s.diags.Add( - diag.SeverityLevelError, - "consul_agent SDs are not currently supported in Grafana Agent Flow - see https://github.com/grafana/agent/issues/2261", - ) - } -} diff --git a/converter/internal/promtailconvert/internal/build/consul_sd.go b/converter/internal/promtailconvert/internal/build/consul_sd.go deleted file mode 100644 index 84bffd1bd3cf..000000000000 --- a/converter/internal/promtailconvert/internal/build/consul_sd.go +++ /dev/null @@ -1,23 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendConsulSDs() { - if len(s.cfg.ServiceDiscoveryConfig.ConsulSDConfigs) == 0 { - return - } - - for i, sd := range s.cfg.ServiceDiscoveryConfig.ConsulSDConfigs { - args := prometheusconvert.ToDiscoveryConsul(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride( - []string{"discovery", "consul"}, - compLabel, - args, - )) - s.allTargetsExps = append(s.allTargetsExps, "discovery.consul."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/digitalocean_sd.go b/converter/internal/promtailconvert/internal/build/digitalocean_sd.go deleted file mode 100644 index acd1c6f385ee..000000000000 --- a/converter/internal/promtailconvert/internal/build/digitalocean_sd.go +++ /dev/null @@ -1,23 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendDigitalOceanSDs() { - if len(s.cfg.ServiceDiscoveryConfig.DigitalOceanSDConfigs) == 0 { - return - } - for i, sd := range s.cfg.ServiceDiscoveryConfig.DigitalOceanSDConfigs { - s.diags.AddAll(prometheusconvert.ValidateDiscoveryDigitalOcean(sd)) - args := prometheusconvert.ToDiscoveryDigitalOcean(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride( - []string{"discovery", "digitalocean"}, - compLabel, - args, - )) - s.allTargetsExps = append(s.allTargetsExps, "discovery.digitalocean."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/docker_sd.go b/converter/internal/promtailconvert/internal/build/docker_sd.go deleted file mode 100644 index 47346c1ab842..000000000000 --- a/converter/internal/promtailconvert/internal/build/docker_sd.go +++ /dev/null @@ -1,23 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendDockerSDs() { - if len(s.cfg.DockerSDConfigs) == 0 { - return - } - for i, sd := range s.cfg.DockerSDConfigs { - s.diags.AddAll(prometheusconvert.ValidateHttpClientConfig(&sd.HTTPClientConfig)) - args := prometheusconvert.ToDiscoveryDocker(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride( - []string{"discovery", "docker"}, - compLabel, - args, - )) - s.allTargetsExps = append(s.allTargetsExps, "discovery.docker."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/ec2_sd.go b/converter/internal/promtailconvert/internal/build/ec2_sd.go deleted file mode 100644 index d4ccb1bb4e10..000000000000 --- a/converter/internal/promtailconvert/internal/build/ec2_sd.go +++ /dev/null @@ -1,23 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendEC2SDs() { - if len(s.cfg.ServiceDiscoveryConfig.EC2SDConfigs) == 0 { - return - } - for i, sd := range s.cfg.ServiceDiscoveryConfig.EC2SDConfigs { - s.diags.AddAll(prometheusconvert.ValidateDiscoveryEC2(sd)) - args := prometheusconvert.ToDiscoveryEC2(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride( - []string{"discovery", "ec2"}, - compLabel, - args, - )) - s.allTargetsExps = append(s.allTargetsExps, "discovery.ec2."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/file_sd.go b/converter/internal/promtailconvert/internal/build/file_sd.go deleted file mode 100644 index 62a6ac8cde97..000000000000 --- a/converter/internal/promtailconvert/internal/build/file_sd.go +++ /dev/null @@ -1,18 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendFileSDs() { - if len(s.cfg.ServiceDiscoveryConfig.FileSDConfigs) == 0 { - return - } - for i, sd := range s.cfg.ServiceDiscoveryConfig.FileSDConfigs { - args := prometheusconvert.ToDiscoveryFile(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride([]string{"discovery", "file"}, compLabel, args)) - s.allTargetsExps = append(s.allTargetsExps, "discovery.file."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/gce_sd.go b/converter/internal/promtailconvert/internal/build/gce_sd.go deleted file mode 100644 index 7cb3a29fd4ab..000000000000 --- a/converter/internal/promtailconvert/internal/build/gce_sd.go +++ /dev/null @@ -1,23 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendGCESDs() { - if len(s.cfg.ServiceDiscoveryConfig.GCESDConfigs) == 0 { - return - } - for i, sd := range s.cfg.ServiceDiscoveryConfig.GCESDConfigs { - s.diags.AddAll(prometheusconvert.ValidateDiscoveryGCE(sd)) - args := prometheusconvert.ToDiscoveryGCE(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride( - []string{"discovery", "gce"}, - compLabel, - args, - )) - s.allTargetsExps = append(s.allTargetsExps, "discovery.gce."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/kubernetes_sd.go b/converter/internal/promtailconvert/internal/build/kubernetes_sd.go deleted file mode 100644 index 1822c46d32ea..000000000000 --- a/converter/internal/promtailconvert/internal/build/kubernetes_sd.go +++ /dev/null @@ -1,24 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/internal/common" - "github.com/grafana/agent/converter/internal/prometheusconvert" -) - -func (s *ScrapeConfigBuilder) AppendKubernetesSDs() { - if len(s.cfg.ServiceDiscoveryConfig.KubernetesSDConfigs) == 0 { - return - } - - for i, sd := range s.cfg.ServiceDiscoveryConfig.KubernetesSDConfigs { - s.diags.AddAll(prometheusconvert.ValidateHttpClientConfig(&sd.HTTPClientConfig)) - args := prometheusconvert.ToDiscoveryKubernetes(sd) - compLabel := common.LabelWithIndex(i, s.globalCtx.LabelPrefix, s.cfg.JobName) - s.f.Body().AppendBlock(common.NewBlockWithOverride( - []string{"discovery", "kubernetes"}, - compLabel, - args, - )) - s.allTargetsExps = append(s.allTargetsExps, "discovery.kubernetes."+compLabel+".targets") - } -} diff --git a/converter/internal/promtailconvert/internal/build/scrape_builder.go b/converter/internal/promtailconvert/internal/build/scrape_builder.go index ba9325e056f0..975da539e446 100644 --- a/converter/internal/promtailconvert/internal/build/scrape_builder.go +++ b/converter/internal/promtailconvert/internal/build/scrape_builder.go @@ -53,27 +53,6 @@ func NewScrapeConfigBuilder( } } -func (s *ScrapeConfigBuilder) Validate() { - if len(s.cfg.ServiceDiscoveryConfig.DockerSwarmSDConfigs) != 0 { - s.diags.Add(diag.SeverityLevelError, "dockerswarm_sd_configs is not supported") - } - if len(s.cfg.ServiceDiscoveryConfig.ServersetSDConfigs) != 0 { - s.diags.Add(diag.SeverityLevelError, "serverset_sd_configs is not supported") - } - if len(s.cfg.ServiceDiscoveryConfig.NerveSDConfigs) != 0 { - s.diags.Add(diag.SeverityLevelError, "nerve_sd_configs is not supported") - } - if len(s.cfg.ServiceDiscoveryConfig.MarathonSDConfigs) != 0 { - s.diags.Add(diag.SeverityLevelError, "marathon_sd_configs is not supported") - } - if len(s.cfg.ServiceDiscoveryConfig.OpenstackSDConfigs) != 0 { - s.diags.Add(diag.SeverityLevelError, "openstack_sd_configs is not supported") - } - if len(s.cfg.ServiceDiscoveryConfig.TritonSDConfigs) != 0 { - s.diags.Add(diag.SeverityLevelError, "triton_sd_configs is not supported") - } -} - func (s *ScrapeConfigBuilder) Sanitize() { var err error s.cfg.JobName, err = scanner.SanitizeIdentifier(s.cfg.JobName) diff --git a/converter/internal/promtailconvert/internal/build/service_discovery.go b/converter/internal/promtailconvert/internal/build/service_discovery.go new file mode 100644 index 000000000000..ab8475894785 --- /dev/null +++ b/converter/internal/promtailconvert/internal/build/service_discovery.go @@ -0,0 +1,121 @@ +package build + +import ( + "github.com/grafana/agent/component/discovery" + "github.com/grafana/agent/converter/diag" + "github.com/grafana/agent/converter/internal/common" + "github.com/grafana/agent/converter/internal/prometheusconvert" + + "github.com/grafana/loki/clients/pkg/promtail/scrapeconfig" + prom_discover "github.com/prometheus/prometheus/discovery" +) + +func (s *ScrapeConfigBuilder) AppendSDs() { + sdConfigs := toDiscoveryConfig(s.cfg) + if len(sdConfigs) == 0 { + return + } + + pb := prometheusconvert.NewPrometheusBlocks() + targets := prometheusconvert.AppendServiceDiscoveryConfigs(pb, sdConfigs, common.LabelForParts(s.globalCtx.LabelPrefix, s.cfg.JobName)) + pb.AppendToFile(s.f) + + targetLiterals := make([]discovery.Target, 0) + for _, target := range targets { + if expr, ok := target["__expr__"]; ok { + // use the expression if __expr__ is set + s.allTargetsExps = append(s.allTargetsExps, expr) + } else { + // otherwise, use the target as a literal + targetLiterals = append(targetLiterals, target) + } + } + + // write the target literals as a string if there are any + if len(targetLiterals) != 0 { + literalsStr, err := toRiverExpression(targetLiterals) + if err != nil { // should not happen, unless we have a bug + s.diags.Add( + diag.SeverityLevelCritical, + "failed to write static SD targets as valid River expression: "+err.Error(), + ) + } + s.allTargetsExps = append(s.allTargetsExps, literalsStr) + } + + s.diags.AddAll(prometheusconvert.ValidateServiceDiscoveryConfigs(sdConfigs)) +} + +func toDiscoveryConfig(cfg *scrapeconfig.Config) prom_discover.Configs { + sdConfigs := make(prom_discover.Configs, 0) + + for _, sd := range cfg.ServiceDiscoveryConfig.AzureSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.ConsulSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.ConsulAgentSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.DigitalOceanSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.DockerSwarmSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.DockerSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.DNSSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.EC2SDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.FileSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.GCESDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.KubernetesSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.MarathonSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.NerveSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.OpenstackSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.ServersetSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + if len(cfg.ServiceDiscoveryConfig.StaticConfigs) != 0 { + sdConfigs = append(sdConfigs, cfg.ServiceDiscoveryConfig.StaticConfigs) + } + + for _, sd := range cfg.ServiceDiscoveryConfig.TritonSDConfigs { + sdConfigs = append(sdConfigs, sd) + } + + return sdConfigs +} diff --git a/converter/internal/promtailconvert/internal/build/static_sd.go b/converter/internal/promtailconvert/internal/build/static_sd.go deleted file mode 100644 index 39a755ea8e05..000000000000 --- a/converter/internal/promtailconvert/internal/build/static_sd.go +++ /dev/null @@ -1,33 +0,0 @@ -package build - -import ( - "github.com/grafana/agent/converter/diag" - "golang.org/x/exp/maps" -) - -func (s *ScrapeConfigBuilder) AppendStaticSDs() { - if len(s.cfg.ServiceDiscoveryConfig.StaticConfigs) == 0 { - return - } - - var allStaticTargets []map[string]string - for _, sd := range s.cfg.ServiceDiscoveryConfig.StaticConfigs { - labels := convertPromLabels(sd.Labels) - for _, target := range sd.Targets { - mergedTarget := map[string]string{} - maps.Copy(mergedTarget, labels) - maps.Copy(mergedTarget, convertPromLabels(target)) - allStaticTargets = append(allStaticTargets, mergedTarget) - } - } - - targetsExpr, err := toRiverExpression(allStaticTargets) - if err != nil { - s.diags.Add( - diag.SeverityLevelCritical, - "failed to write static SD targets as valid River expression: "+err.Error(), - ) - } - - s.allTargetsExps = append(s.allTargetsExps, targetsExpr) -} diff --git a/converter/internal/promtailconvert/promtailconvert.go b/converter/internal/promtailconvert/promtailconvert.go index 0762e085dba5..87fe74b427f0 100644 --- a/converter/internal/promtailconvert/promtailconvert.go +++ b/converter/internal/promtailconvert/promtailconvert.go @@ -132,20 +132,10 @@ func appendScrapeConfig( ) { b := build.NewScrapeConfigBuilder(f, diags, cfg, gctx) - b.Validate() b.Sanitize() // Append all the SD components - b.AppendKubernetesSDs() - b.AppendDockerSDs() - b.AppendStaticSDs() - b.AppendFileSDs() - b.AppendConsulSDs() - b.AppendConsulAgentSDs() - b.AppendDigitalOceanSDs() - b.AppendGCESDs() - b.AppendEC2SDs() - b.AppendAzureSDs() + b.AppendSDs() // Append loki.source.file to process all SD components' targets. // If any relabelling is required, it will be done via a discovery.relabel component. diff --git a/converter/internal/promtailconvert/testdata/unsupported.diags b/converter/internal/promtailconvert/testdata/unsupported.diags index 798cabfa4561..a433eff3b633 100644 --- a/converter/internal/promtailconvert/testdata/unsupported.diags +++ b/converter/internal/promtailconvert/testdata/unsupported.diags @@ -10,9 +10,9 @@ (Error) server.http_path_prefix is not supported (Warning) server.health_check_target disabling is not supported in Flow mode (Warning) stream_lag_labels is deprecated and the associated metric has been removed -(Error) dockerswarm_sd_configs is not supported -(Error) serverset_sd_configs is not supported -(Error) nerve_sd_configs is not supported -(Error) marathon_sd_configs is not supported -(Error) openstack_sd_configs is not supported -(Error) triton_sd_configs is not supported \ No newline at end of file +(Error) unsupported service discovery dockerswarm was provided +(Error) unsupported service discovery marathon was provided +(Error) unsupported service discovery nerve was provided +(Error) unsupported service discovery openstack was provided +(Error) unsupported service discovery serverset was provided +(Error) unsupported service discovery triton was provided \ No newline at end of file diff --git a/converter/internal/staticconvert/testdata/prom_scrape.river b/converter/internal/staticconvert/testdata/prom_scrape.river index cac2d4cf5656..777a13fbe5ee 100644 --- a/converter/internal/staticconvert/testdata/prom_scrape.river +++ b/converter/internal/staticconvert/testdata/prom_scrape.river @@ -34,8 +34,8 @@ discovery.azure "metrics_agent_promobee_2" { discovery.relabel "metrics_agent_promobee" { targets = concat( - discovery.azure.metrics_agent_promobee_2.targets, discovery.azure.metrics_agent_promobee.targets, + discovery.azure.metrics_agent_promobee_2.targets, [{ __address__ = "localhost:9090", }], @@ -60,7 +60,7 @@ discovery.relabel "metrics_agent_promobee" { prometheus.scrape "metrics_agent_prometheus" { targets = concat( [{ - __address__ = "localhost:9101", + __address__ = "localhost:9099", }], [{ __address__ = "localhost:9101", diff --git a/converter/internal/staticconvert/testdata/promtail_prom.river b/converter/internal/staticconvert/testdata/promtail_prom.river index 591b47c25494..f3b810dbe704 100644 --- a/converter/internal/staticconvert/testdata/promtail_prom.river +++ b/converter/internal/staticconvert/testdata/promtail_prom.river @@ -8,7 +8,7 @@ prometheus.scrape "metrics_name_jobName" { targets = concat( discovery.consul.metrics_name_jobName.targets, [{ - __address__ = "localhost:9101", + __address__ = "localhost:9099", }], [{ __address__ = "localhost:9101",