refactor promtail converter sd configs so they reuse prometheus conve… (

#5069) * refactor promtail converter sd configs so they reuse prometheus converter functionality Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * cleanup words Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * unexport a bunch of stuff that isn't needed outside the prometheusconvert package Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * don't do SD targets in backwards order during converter Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> * get rid of static SD boilerplate --------- Signed-off-by: erikbaranowski <39704712+erikbaranowski@users.noreply.github.com> Co-authored-by: Piotr Gwizdala <17101802+thampiotr@users.noreply.github.com>
grafana · Sep 5, 2023 · 89b334e · 89b334e
1 parent 3fa4983
commit 89b334e
Show file tree

Hide file tree

Showing 30 changed files with 201 additions and 326 deletions.
diff --git a/converter/internal/prometheusconvert/azure.go b/converter/internal/prometheusconvert/azure.go
@@ -13,14 +13,14 @@ import (
 )
 
 func appendDiscoveryAzure(pb *prometheusBlocks, label string, sdConfig *prom_azure.SDConfig) discovery.Exports {
-	discoveryAzureArgs := ToDiscoveryAzure(sdConfig)
+	discoveryAzureArgs := toDiscoveryAzure(sdConfig)
 	name := []string{"discovery", "azure"}
 	block := common.NewBlockWithOverride(name, label, discoveryAzureArgs)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
 	return NewDiscoveryExports("discovery.azure." + label + ".targets")
 }
 
-func ToDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments {
+func toDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments {
 	if sdConfig == nil {
 		return nil
 	}
@@ -40,7 +40,7 @@ func ToDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments {
 	}
 }
 
-func ValidateDiscoveryAzure(sdConfig *prom_azure.SDConfig) diag.Diagnostics {
+func validateDiscoveryAzure(sdConfig *prom_azure.SDConfig) diag.Diagnostics {
 	return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
 }
 

diff --git a/converter/internal/prometheusconvert/consul.go b/converter/internal/prometheusconvert/consul.go
@@ -12,7 +12,7 @@ import (
 )
 
 func appendDiscoveryConsul(pb *prometheusBlocks, label string, sdConfig *prom_consul.SDConfig) discovery.Exports {
-	discoveryConsulArgs := ToDiscoveryConsul(sdConfig)
+	discoveryConsulArgs := toDiscoveryConsul(sdConfig)
 	name := []string{"discovery", "consul"}
 	block := common.NewBlockWithOverride(name, label, discoveryConsulArgs)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
@@ -23,7 +23,7 @@ func validateDiscoveryConsul(sdConfig *prom_consul.SDConfig) diag.Diagnostics {
 	return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
 }
 
-func ToDiscoveryConsul(sdConfig *prom_consul.SDConfig) *consul.Arguments {
+func toDiscoveryConsul(sdConfig *prom_consul.SDConfig) *consul.Arguments {
 	if sdConfig == nil {
 		return nil
 	}

diff --git a/converter/internal/prometheusconvert/digitalocean.go b/converter/internal/prometheusconvert/digitalocean.go
@@ -15,14 +15,14 @@ import (
 )
 
 func appendDiscoveryDigitalOcean(pb *prometheusBlocks, label string, sdConfig *prom_digitalocean.SDConfig) discovery.Exports {
-	discoveryDigitalOceanArgs := ToDiscoveryDigitalOcean(sdConfig)
+	discoveryDigitalOceanArgs := toDiscoveryDigitalOcean(sdConfig)
 	name := []string{"discovery", "digitalocean"}
 	block := common.NewBlockWithOverride(name, label, discoveryDigitalOceanArgs)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
 	return NewDiscoveryExports("discovery.digitalocean." + label + ".targets")
 }
 
-func ValidateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Diagnostics {
+func validateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Diagnostics {
 	var diags diag.Diagnostics
 
 	if sdConfig.HTTPClientConfig.BasicAuth != nil {
@@ -46,7 +46,7 @@ func ValidateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Di
 	return diags
 }
 
-func ToDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) *digitalocean.Arguments {
+func toDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) *digitalocean.Arguments {
 	if sdConfig == nil {
 		return nil
 	}

diff --git a/converter/internal/prometheusconvert/docker.go b/converter/internal/prometheusconvert/docker.go
@@ -11,7 +11,7 @@ import (
 )
 
 func appendDiscoveryDocker(pb *prometheusBlocks, label string, sdConfig *prom_docker.DockerSDConfig) discovery.Exports {
-	discoveryDockerArgs := ToDiscoveryDocker(sdConfig)
+	discoveryDockerArgs := toDiscoveryDocker(sdConfig)
 	name := []string{"discovery", "docker"}
 	block := common.NewBlockWithOverride(name, label, discoveryDockerArgs)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
@@ -22,7 +22,7 @@ func validateDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) diag.Diagnost
 	return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
 }
 
-func ToDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) *docker.Arguments {
+func toDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) *docker.Arguments {
 	if sdConfig == nil {
 		return nil
 	}

diff --git a/converter/internal/prometheusconvert/ec2.go b/converter/internal/prometheusconvert/ec2.go
@@ -14,14 +14,14 @@ import (
 )
 
 func appendDiscoveryEC2(pb *prometheusBlocks, label string, sdConfig *prom_aws.EC2SDConfig) discovery.Exports {
-	discoveryec2Args := ToDiscoveryEC2(sdConfig)
+	discoveryec2Args := toDiscoveryEC2(sdConfig)
 	name := []string{"discovery", "ec2"}
 	block := common.NewBlockWithOverride(name, label, discoveryec2Args)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
 	return NewDiscoveryExports("discovery.ec2." + label + ".targets")
 }
 
-func ValidateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics {
+func validateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics {
 	var diags diag.Diagnostics
 
 	if sdConfig.HTTPClientConfig.BasicAuth != nil {
@@ -64,7 +64,7 @@ func ValidateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics {
 	return diags
 }
 
-func ToDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) *aws.EC2Arguments {
+func toDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) *aws.EC2Arguments {
 	if sdConfig == nil {
 		return nil
 	}

diff --git a/converter/internal/prometheusconvert/file.go b/converter/internal/prometheusconvert/file.go
@@ -11,7 +11,7 @@ import (
 )
 
 func appendDiscoveryFile(pb *prometheusBlocks, label string, sdConfig *prom_file.SDConfig) discovery.Exports {
-	discoveryFileArgs := ToDiscoveryFile(sdConfig)
+	discoveryFileArgs := toDiscoveryFile(sdConfig)
 	name := []string{"discovery", "file"}
 	block := common.NewBlockWithOverride(name, label, discoveryFileArgs)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
@@ -22,7 +22,7 @@ func validateDiscoveryFile(sdConfig *prom_file.SDConfig) diag.Diagnostics {
 	return make(diag.Diagnostics, 0)
 }
 
-func ToDiscoveryFile(sdConfig *prom_file.SDConfig) *file.Arguments {
+func toDiscoveryFile(sdConfig *prom_file.SDConfig) *file.Arguments {
 	if sdConfig == nil {
 		return nil
 	}

diff --git a/converter/internal/prometheusconvert/gce.go b/converter/internal/prometheusconvert/gce.go
@@ -11,18 +11,18 @@ import (
 )
 
 func appendDiscoveryGCE(pb *prometheusBlocks, label string, sdConfig *prom_gce.SDConfig) discovery.Exports {
-	discoveryGCEArgs := ToDiscoveryGCE(sdConfig)
+	discoveryGCEArgs := toDiscoveryGCE(sdConfig)
 	name := []string{"discovery", "gce"}
 	block := common.NewBlockWithOverride(name, label, discoveryGCEArgs)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
 	return NewDiscoveryExports("discovery.gce." + label + ".targets")
 }
 
-func ValidateDiscoveryGCE(sdConfig *prom_gce.SDConfig) diag.Diagnostics {
+func validateDiscoveryGCE(sdConfig *prom_gce.SDConfig) diag.Diagnostics {
 	return make(diag.Diagnostics, 0)
 }
 
-func ToDiscoveryGCE(sdConfig *prom_gce.SDConfig) *gce.Arguments {
+func toDiscoveryGCE(sdConfig *prom_gce.SDConfig) *gce.Arguments {
 	if sdConfig == nil {
 		return nil
 	}

diff --git a/converter/internal/prometheusconvert/kubernetes.go b/converter/internal/prometheusconvert/kubernetes.go
@@ -10,7 +10,7 @@ import (
 )
 
 func appendDiscoveryKubernetes(pb *prometheusBlocks, label string, sdConfig *prom_kubernetes.SDConfig) discovery.Exports {
-	discoveryKubernetesArgs := ToDiscoveryKubernetes(sdConfig)
+	discoveryKubernetesArgs := toDiscoveryKubernetes(sdConfig)
 	name := []string{"discovery", "kubernetes"}
 	block := common.NewBlockWithOverride(name, label, discoveryKubernetesArgs)
 	pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
@@ -21,7 +21,7 @@ func validateDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) diag.Diagno
 	return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
 }
 
-func ToDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) *kubernetes.Arguments {
+func toDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) *kubernetes.Arguments {
 	if sdConfig == nil {
 		return nil
 	}

diff --git a/converter/internal/prometheusconvert/prometheus_blocks.go b/converter/internal/prometheusconvert/prometheus_blocks.go
@@ -19,7 +19,7 @@ type prometheusBlocks struct {
 	prometheusRemoteWriteBlocks []prometheusBlock
 }
 
-func newPrometheusBlocks() *prometheusBlocks {
+func NewPrometheusBlocks() *prometheusBlocks {
 	return &prometheusBlocks{
 		discoveryBlocks:             []prometheusBlock{},
 		discoveryRelabelBlocks:      []prometheusBlock{},
@@ -29,15 +29,15 @@ func newPrometheusBlocks() *prometheusBlocks {
 	}
 }
 
-// appendToFile attaches prometheus blocks in a specific order.
+// AppendToFile attaches prometheus blocks in a specific order.
 //
 // Order of blocks:
 // 1. Discovery component(s)
 // 2. Discovery relabel component(s) (if any)
 // 3. Prometheus scrape component(s)
 // 4. Prometheus relabel component(s) (if any)
 // 5. Prometheus remote_write
-func (pb *prometheusBlocks) appendToFile(f *builder.File) {
+func (pb *prometheusBlocks) AppendToFile(f *builder.File) {
 	for _, promBlock := range pb.discoveryBlocks {
 		f.Body().AppendBlock(promBlock.block)
 	}

diff --git a/converter/internal/prometheusconvert/prometheusconvert.go b/converter/internal/prometheusconvert/prometheusconvert.go
@@ -69,7 +69,7 @@ func AppendAll(f *builder.File, promConfig *prom_config.Config) diag.Diagnostics
 // pipeline. Additional options can be provided overriding the job name, extra
 // scrape targets, and predefined remote write exports.
 func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToCompLabelsFunc func(string) string, extraScrapeTargets []discovery.Target, remoteWriteExports *remotewrite.Exports) diag.Diagnostics {
-	pb := newPrometheusBlocks()
+	pb := NewPrometheusBlocks()
 
 	if remoteWriteExports == nil {
 		labelPrefix := ""
@@ -92,7 +92,7 @@ func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToC
 			scrapeForwardTo = []storage.Appendable{promMetricsRelabelExports.Receiver}
 		}
 
-		scrapeTargets := appendServiceDiscoveryConfigs(pb, scrapeConfig.ServiceDiscoveryConfigs, label)
+		scrapeTargets := AppendServiceDiscoveryConfigs(pb, scrapeConfig.ServiceDiscoveryConfigs, label)
 		scrapeTargets = append(scrapeTargets, extraScrapeTargets...)
 
 		promDiscoveryRelabelExports := appendDiscoveryRelabel(pb, scrapeConfig.RelabelConfigs, scrapeTargets, label)
@@ -106,15 +106,15 @@ func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToC
 	diags := validate(promConfig)
 	diags.AddAll(pb.getScrapeInfo())
 
-	pb.appendToFile(f)
+	pb.AppendToFile(f)
 
 	return diags
 }
 
-// appendServiceDiscoveryConfigs will loop through the service discovery
+// AppendServiceDiscoveryConfigs will loop through the service discovery
 // configs and append them to the file. This returns the scrape targets
 // and discovery targets as a result.
-func appendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig prom_discover.Configs, label string) []discovery.Target {
+func AppendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig prom_discover.Configs, label string) []discovery.Target {
 	var targets []discovery.Target
 	labelCounts := make(map[string]int)
 	for _, serviceDiscoveryConfig := range serviceDiscoveryConfig {
@@ -154,7 +154,7 @@ func appendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig
 			exports = appendDiscoveryLightsail(pb, common.LabelWithIndex(labelCounts["lightsail"]-1, label), sdc)
 		}
 
-		targets = append(exports.Targets, targets...)
+		targets = append(targets, exports.Targets...)
 	}
 
 	return targets

diff --git a/converter/internal/prometheusconvert/scrape.go b/converter/internal/prometheusconvert/scrape.go
@@ -5,6 +5,8 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/exp/maps"
+
 	"github.com/grafana/agent/component/discovery"
 	"github.com/grafana/agent/component/prometheus/scrape"
 	"github.com/grafana/agent/converter/diag"
@@ -68,7 +70,9 @@ func getScrapeTargets(staticConfig prom_discovery.StaticConfig) []discovery.Targ
 		for _, labelSet := range target.Targets {
 			for labelName, labelValue := range labelSet {
 				targetMap[string(labelName)] = string(labelValue)
-				targets = append(targets, targetMap)
+				newMap := map[string]string{}
+				maps.Copy(newMap, targetMap)
+				targets = append(targets, newMap)
 			}
 		}
 	}

diff --git a/converter/internal/prometheusconvert/testdata/discovery.river b/converter/internal/prometheusconvert/testdata/discovery.river
@@ -32,8 +32,8 @@ discovery.azure "prometheus1_2" {
 
 discovery.relabel "prometheus1" {
 	targets = concat(
-		discovery.azure.prometheus1_2.targets,
 		discovery.azure.prometheus1.targets,
+		discovery.azure.prometheus1_2.targets,
 		[{
 			__address__ = "localhost:9090",
 		}],

diff --git a/converter/internal/prometheusconvert/testdata/scrape.river b/converter/internal/prometheusconvert/testdata/scrape.river
@@ -1,7 +1,7 @@
 prometheus.scrape "prometheus1" {
 	targets = concat(
 		[{
-			__address__ = "localhost:9091",
+			__address__ = "localhost:9090",
 			app         = "foo",
 		}],
 		[{

diff --git a/converter/internal/prometheusconvert/validate.go b/converter/internal/prometheusconvert/validate.go
@@ -69,34 +69,41 @@ func validateScrapeConfigs(scrapeConfigs []*prom_config.ScrapeConfig) diag.Diagn
 
 	for _, scrapeConfig := range scrapeConfigs {
 		diags.AddAll(validatePrometheusScrape(scrapeConfig))
+		diags.AddAll(ValidateServiceDiscoveryConfigs(scrapeConfig.ServiceDiscoveryConfigs))
+	}
+
+	return diags
+}
+
+func ValidateServiceDiscoveryConfigs(serviceDiscoveryConfigs prom_discover.Configs) diag.Diagnostics {
+	var diags diag.Diagnostics
 
-		for _, serviceDiscoveryConfig := range scrapeConfig.ServiceDiscoveryConfigs {
-			switch sdc := serviceDiscoveryConfig.(type) {
-			case prom_discover.StaticConfig:
-				diags.AddAll(validateScrapeTargets(sdc))
-			case *prom_azure.SDConfig:
-				diags.AddAll(ValidateDiscoveryAzure(sdc))
-			case *prom_consul.SDConfig:
-				diags.AddAll(validateDiscoveryConsul(sdc))
-			case *prom_digitalocean.SDConfig:
-				diags.AddAll(ValidateDiscoveryDigitalOcean(sdc))
-			case *prom_dns.SDConfig:
-				diags.AddAll(validateDiscoveryDns(sdc))
-			case *prom_docker.DockerSDConfig:
-				diags.AddAll(validateDiscoveryDocker(sdc))
-			case *prom_aws.EC2SDConfig:
-				diags.AddAll(ValidateDiscoveryEC2(sdc))
-			case *prom_file.SDConfig:
-				diags.AddAll(validateDiscoveryFile(sdc))
-			case *prom_gce.SDConfig:
-				diags.AddAll(ValidateDiscoveryGCE(sdc))
-			case *prom_kubernetes.SDConfig:
-				diags.AddAll(validateDiscoveryKubernetes(sdc))
-			case *prom_aws.LightsailSDConfig:
-				diags.AddAll(validateDiscoveryLightsail(sdc))
-			default:
-				diags.Add(diag.SeverityLevelError, fmt.Sprintf("unsupported service discovery %s was provided", serviceDiscoveryConfig.Name()))
-			}
+	for _, serviceDiscoveryConfig := range serviceDiscoveryConfigs {
+		switch sdc := serviceDiscoveryConfig.(type) {
+		case prom_discover.StaticConfig:
+			diags.AddAll(validateScrapeTargets(sdc))
+		case *prom_azure.SDConfig:
+			diags.AddAll(validateDiscoveryAzure(sdc))
+		case *prom_consul.SDConfig:
+			diags.AddAll(validateDiscoveryConsul(sdc))
+		case *prom_digitalocean.SDConfig:
+			diags.AddAll(validateDiscoveryDigitalOcean(sdc))
+		case *prom_dns.SDConfig:
+			diags.AddAll(validateDiscoveryDns(sdc))
+		case *prom_docker.DockerSDConfig:
+			diags.AddAll(validateDiscoveryDocker(sdc))
+		case *prom_aws.EC2SDConfig:
+			diags.AddAll(validateDiscoveryEC2(sdc))
+		case *prom_file.SDConfig:
+			diags.AddAll(validateDiscoveryFile(sdc))
+		case *prom_gce.SDConfig:
+			diags.AddAll(validateDiscoveryGCE(sdc))
+		case *prom_kubernetes.SDConfig:
+			diags.AddAll(validateDiscoveryKubernetes(sdc))
+		case *prom_aws.LightsailSDConfig:
+			diags.AddAll(validateDiscoveryLightsail(sdc))
+		default:
+			diags.Add(diag.SeverityLevelError, fmt.Sprintf("unsupported service discovery %s was provided", serviceDiscoveryConfig.Name()))
 		}
 	}
 

diff --git a/converter/internal/promtailconvert/internal/build/azure_sd.go b/converter/internal/promtailconvert/internal/build/azure_sd.go
diff --git a/converter/internal/promtailconvert/internal/build/consul_agent.go b/converter/internal/promtailconvert/internal/build/consul_agent.go