Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor promtail converter sd configs so they reuse prometheus conve… #5069

Merged
merged 5 commits into from
Sep 5, 2023
Merged

refactor promtail converter sd configs so they reuse prometheus conve… #5069

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
3732667
refactor promtail converter sd configs so they reuse prometheus conve…
erikbaranowski Sep 1, 2023
8625b5f
cleanup words
erikbaranowski Sep 1, 2023
c979702
unexport a bunch of stuff that isn't needed outside the prometheuscon…
erikbaranowski Sep 1, 2023
b990eb5
don't do SD targets in backwards order during converter
erikbaranowski Sep 1, 2023
e283f00
get rid of static SD boilerplate
thampiotr Sep 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions converter/internal/prometheusconvert/azure.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ import (
)

func appendDiscoveryAzure(pb *prometheusBlocks, label string, sdConfig *prom_azure.SDConfig) discovery.Exports {
discoveryAzureArgs := ToDiscoveryAzure(sdConfig)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these lower level functions do not need be public anymore. we can bring this back later if we need to.

discoveryAzureArgs := toDiscoveryAzure(sdConfig)
name := []string{"discovery", "azure"}
block := common.NewBlockWithOverride(name, label, discoveryAzureArgs)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
return NewDiscoveryExports("discovery.azure." + label + ".targets")
}

func ToDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments {
func toDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments {
if sdConfig == nil {
return nil
}
Expand All @@ -40,7 +40,7 @@ func ToDiscoveryAzure(sdConfig *prom_azure.SDConfig) *azure.Arguments {
}
}

func ValidateDiscoveryAzure(sdConfig *prom_azure.SDConfig) diag.Diagnostics {
func validateDiscoveryAzure(sdConfig *prom_azure.SDConfig) diag.Diagnostics {
return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
}

Expand Down
4 changes: 2 additions & 2 deletions converter/internal/prometheusconvert/consul.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (
)

func appendDiscoveryConsul(pb *prometheusBlocks, label string, sdConfig *prom_consul.SDConfig) discovery.Exports {
discoveryConsulArgs := ToDiscoveryConsul(sdConfig)
discoveryConsulArgs := toDiscoveryConsul(sdConfig)
name := []string{"discovery", "consul"}
block := common.NewBlockWithOverride(name, label, discoveryConsulArgs)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
Expand All @@ -23,7 +23,7 @@ func validateDiscoveryConsul(sdConfig *prom_consul.SDConfig) diag.Diagnostics {
return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
}

func ToDiscoveryConsul(sdConfig *prom_consul.SDConfig) *consul.Arguments {
func toDiscoveryConsul(sdConfig *prom_consul.SDConfig) *consul.Arguments {
if sdConfig == nil {
return nil
}
Expand Down
6 changes: 3 additions & 3 deletions converter/internal/prometheusconvert/digitalocean.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ import (
)

func appendDiscoveryDigitalOcean(pb *prometheusBlocks, label string, sdConfig *prom_digitalocean.SDConfig) discovery.Exports {
discoveryDigitalOceanArgs := ToDiscoveryDigitalOcean(sdConfig)
discoveryDigitalOceanArgs := toDiscoveryDigitalOcean(sdConfig)
name := []string{"discovery", "digitalocean"}
block := common.NewBlockWithOverride(name, label, discoveryDigitalOceanArgs)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
return NewDiscoveryExports("discovery.digitalocean." + label + ".targets")
}

func ValidateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Diagnostics {
func validateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Diagnostics {
var diags diag.Diagnostics

if sdConfig.HTTPClientConfig.BasicAuth != nil {
Expand All @@ -46,7 +46,7 @@ func ValidateDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) diag.Di
return diags
}

func ToDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) *digitalocean.Arguments {
func toDiscoveryDigitalOcean(sdConfig *prom_digitalocean.SDConfig) *digitalocean.Arguments {
if sdConfig == nil {
return nil
}
Expand Down
4 changes: 2 additions & 2 deletions converter/internal/prometheusconvert/docker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
)

func appendDiscoveryDocker(pb *prometheusBlocks, label string, sdConfig *prom_docker.DockerSDConfig) discovery.Exports {
discoveryDockerArgs := ToDiscoveryDocker(sdConfig)
discoveryDockerArgs := toDiscoveryDocker(sdConfig)
name := []string{"discovery", "docker"}
block := common.NewBlockWithOverride(name, label, discoveryDockerArgs)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
Expand All @@ -22,7 +22,7 @@ func validateDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) diag.Diagnost
return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
}

func ToDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) *docker.Arguments {
func toDiscoveryDocker(sdConfig *prom_docker.DockerSDConfig) *docker.Arguments {
if sdConfig == nil {
return nil
}
Expand Down
6 changes: 3 additions & 3 deletions converter/internal/prometheusconvert/ec2.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@ import (
)

func appendDiscoveryEC2(pb *prometheusBlocks, label string, sdConfig *prom_aws.EC2SDConfig) discovery.Exports {
discoveryec2Args := ToDiscoveryEC2(sdConfig)
discoveryec2Args := toDiscoveryEC2(sdConfig)
name := []string{"discovery", "ec2"}
block := common.NewBlockWithOverride(name, label, discoveryec2Args)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
return NewDiscoveryExports("discovery.ec2." + label + ".targets")
}

func ValidateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics {
func validateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics {
var diags diag.Diagnostics

if sdConfig.HTTPClientConfig.BasicAuth != nil {
Expand Down Expand Up @@ -64,7 +64,7 @@ func ValidateDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) diag.Diagnostics {
return diags
}

func ToDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) *aws.EC2Arguments {
func toDiscoveryEC2(sdConfig *prom_aws.EC2SDConfig) *aws.EC2Arguments {
if sdConfig == nil {
return nil
}
Expand Down
4 changes: 2 additions & 2 deletions converter/internal/prometheusconvert/file.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
)

func appendDiscoveryFile(pb *prometheusBlocks, label string, sdConfig *prom_file.SDConfig) discovery.Exports {
discoveryFileArgs := ToDiscoveryFile(sdConfig)
discoveryFileArgs := toDiscoveryFile(sdConfig)
name := []string{"discovery", "file"}
block := common.NewBlockWithOverride(name, label, discoveryFileArgs)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
Expand All @@ -22,7 +22,7 @@ func validateDiscoveryFile(sdConfig *prom_file.SDConfig) diag.Diagnostics {
return make(diag.Diagnostics, 0)
}

func ToDiscoveryFile(sdConfig *prom_file.SDConfig) *file.Arguments {
func toDiscoveryFile(sdConfig *prom_file.SDConfig) *file.Arguments {
if sdConfig == nil {
return nil
}
Expand Down
6 changes: 3 additions & 3 deletions converter/internal/prometheusconvert/gce.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,18 +11,18 @@ import (
)

func appendDiscoveryGCE(pb *prometheusBlocks, label string, sdConfig *prom_gce.SDConfig) discovery.Exports {
discoveryGCEArgs := ToDiscoveryGCE(sdConfig)
discoveryGCEArgs := toDiscoveryGCE(sdConfig)
name := []string{"discovery", "gce"}
block := common.NewBlockWithOverride(name, label, discoveryGCEArgs)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
return NewDiscoveryExports("discovery.gce." + label + ".targets")
}

func ValidateDiscoveryGCE(sdConfig *prom_gce.SDConfig) diag.Diagnostics {
func validateDiscoveryGCE(sdConfig *prom_gce.SDConfig) diag.Diagnostics {
return make(diag.Diagnostics, 0)
}

func ToDiscoveryGCE(sdConfig *prom_gce.SDConfig) *gce.Arguments {
func toDiscoveryGCE(sdConfig *prom_gce.SDConfig) *gce.Arguments {
if sdConfig == nil {
return nil
}
Expand Down
4 changes: 2 additions & 2 deletions converter/internal/prometheusconvert/kubernetes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ import (
)

func appendDiscoveryKubernetes(pb *prometheusBlocks, label string, sdConfig *prom_kubernetes.SDConfig) discovery.Exports {
discoveryKubernetesArgs := ToDiscoveryKubernetes(sdConfig)
discoveryKubernetesArgs := toDiscoveryKubernetes(sdConfig)
name := []string{"discovery", "kubernetes"}
block := common.NewBlockWithOverride(name, label, discoveryKubernetesArgs)
pb.discoveryBlocks = append(pb.discoveryBlocks, newPrometheusBlock(block, name, label, "", ""))
Expand All @@ -21,7 +21,7 @@ func validateDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) diag.Diagno
return ValidateHttpClientConfig(&sdConfig.HTTPClientConfig)
}

func ToDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) *kubernetes.Arguments {
func toDiscoveryKubernetes(sdConfig *prom_kubernetes.SDConfig) *kubernetes.Arguments {
if sdConfig == nil {
return nil
}
Expand Down
6 changes: 3 additions & 3 deletions converter/internal/prometheusconvert/prometheus_blocks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ type prometheusBlocks struct {
prometheusRemoteWriteBlocks []prometheusBlock
}

func newPrometheusBlocks() *prometheusBlocks {
func NewPrometheusBlocks() *prometheusBlocks {
return &prometheusBlocks{
discoveryBlocks: []prometheusBlock{},
discoveryRelabelBlocks: []prometheusBlock{},
Expand All @@ -29,15 +29,15 @@ func newPrometheusBlocks() *prometheusBlocks {
}
}

// appendToFile attaches prometheus blocks in a specific order.
// AppendToFile attaches prometheus blocks in a specific order.
//
// Order of blocks:
// 1. Discovery component(s)
// 2. Discovery relabel component(s) (if any)
// 3. Prometheus scrape component(s)
// 4. Prometheus relabel component(s) (if any)
// 5. Prometheus remote_write
func (pb *prometheusBlocks) appendToFile(f *builder.File) {
func (pb *prometheusBlocks) AppendToFile(f *builder.File) {
for _, promBlock := range pb.discoveryBlocks {
f.Body().AppendBlock(promBlock.block)
}
Expand Down
12 changes: 6 additions & 6 deletions converter/internal/prometheusconvert/prometheusconvert.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func AppendAll(f *builder.File, promConfig *prom_config.Config) diag.Diagnostics
// pipeline. Additional options can be provided overriding the job name, extra
// scrape targets, and predefined remote write exports.
func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToCompLabelsFunc func(string) string, extraScrapeTargets []discovery.Target, remoteWriteExports *remotewrite.Exports) diag.Diagnostics {
pb := newPrometheusBlocks()
pb := NewPrometheusBlocks()

if remoteWriteExports == nil {
labelPrefix := ""
Expand All @@ -92,7 +92,7 @@ func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToC
scrapeForwardTo = []storage.Appendable{promMetricsRelabelExports.Receiver}
}

scrapeTargets := appendServiceDiscoveryConfigs(pb, scrapeConfig.ServiceDiscoveryConfigs, label)
scrapeTargets := AppendServiceDiscoveryConfigs(pb, scrapeConfig.ServiceDiscoveryConfigs, label)
scrapeTargets = append(scrapeTargets, extraScrapeTargets...)

promDiscoveryRelabelExports := appendDiscoveryRelabel(pb, scrapeConfig.RelabelConfigs, scrapeTargets, label)
Expand All @@ -106,15 +106,15 @@ func AppendAllNested(f *builder.File, promConfig *prom_config.Config, jobNameToC
diags := validate(promConfig)
diags.AddAll(pb.getScrapeInfo())

pb.appendToFile(f)
pb.AppendToFile(f)

return diags
}

// appendServiceDiscoveryConfigs will loop through the service discovery
// AppendServiceDiscoveryConfigs will loop through the service discovery
// configs and append them to the file. This returns the scrape targets
// and discovery targets as a result.
func appendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig prom_discover.Configs, label string) []discovery.Target {
func AppendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig prom_discover.Configs, label string) []discovery.Target {
var targets []discovery.Target
labelCounts := make(map[string]int)
for _, serviceDiscoveryConfig := range serviceDiscoveryConfig {
Expand Down Expand Up @@ -154,7 +154,7 @@ func appendServiceDiscoveryConfigs(pb *prometheusBlocks, serviceDiscoveryConfig
exports = appendDiscoveryLightsail(pb, common.LabelWithIndex(labelCounts["lightsail"]-1, label), sdc)
}

targets = append(exports.Targets, targets...)
targets = append(targets, exports.Targets...)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This fixes a backwards ordering that was not a functional problem but weird. Discovered when leveraging this for promtail.

}

return targets
Expand Down
6 changes: 5 additions & 1 deletion converter/internal/prometheusconvert/scrape.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"strings"
"time"

"golang.org/x/exp/maps"

"github.com/grafana/agent/component/discovery"
"github.com/grafana/agent/component/prometheus/scrape"
"github.com/grafana/agent/converter/diag"
Expand Down Expand Up @@ -68,7 +70,9 @@ func getScrapeTargets(staticConfig prom_discovery.StaticConfig) []discovery.Targ
for _, labelSet := range target.Targets {
for labelName, labelValue := range labelSet {
targetMap[string(labelName)] = string(labelValue)
targets = append(targets, targetMap)
newMap := map[string]string{}
maps.Copy(newMap, targetMap)
targets = append(targets, newMap)
Comment on lines +73 to +75
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was a subtle bug where the targetMap was being mutated. We need a copy here. This is why some test cases have the __address__ label changed. I believe this is the correct behaviour now.

}
}
}
Expand Down
2 changes: 1 addition & 1 deletion converter/internal/prometheusconvert/testdata/discovery.river
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ discovery.azure "prometheus1_2" {

discovery.relabel "prometheus1" {
targets = concat(
discovery.azure.prometheus1_2.targets,
discovery.azure.prometheus1.targets,
discovery.azure.prometheus1_2.targets,
[{
__address__ = "localhost:9090",
}],
Expand Down
2 changes: 1 addition & 1 deletion converter/internal/prometheusconvert/testdata/scrape.river
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
prometheus.scrape "prometheus1" {
targets = concat(
[{
__address__ = "localhost:9091",
__address__ = "localhost:9090",
app = "foo",
}],
[{
Expand Down
61 changes: 34 additions & 27 deletions converter/internal/prometheusconvert/validate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,34 +69,41 @@ func validateScrapeConfigs(scrapeConfigs []*prom_config.ScrapeConfig) diag.Diagn

for _, scrapeConfig := range scrapeConfigs {
diags.AddAll(validatePrometheusScrape(scrapeConfig))
diags.AddAll(ValidateServiceDiscoveryConfigs(scrapeConfig.ServiceDiscoveryConfigs))
}

return diags
}

func ValidateServiceDiscoveryConfigs(serviceDiscoveryConfigs prom_discover.Configs) diag.Diagnostics {
var diags diag.Diagnostics

for _, serviceDiscoveryConfig := range scrapeConfig.ServiceDiscoveryConfigs {
switch sdc := serviceDiscoveryConfig.(type) {
case prom_discover.StaticConfig:
diags.AddAll(validateScrapeTargets(sdc))
case *prom_azure.SDConfig:
diags.AddAll(ValidateDiscoveryAzure(sdc))
case *prom_consul.SDConfig:
diags.AddAll(validateDiscoveryConsul(sdc))
case *prom_digitalocean.SDConfig:
diags.AddAll(ValidateDiscoveryDigitalOcean(sdc))
case *prom_dns.SDConfig:
diags.AddAll(validateDiscoveryDns(sdc))
case *prom_docker.DockerSDConfig:
diags.AddAll(validateDiscoveryDocker(sdc))
case *prom_aws.EC2SDConfig:
diags.AddAll(ValidateDiscoveryEC2(sdc))
case *prom_file.SDConfig:
diags.AddAll(validateDiscoveryFile(sdc))
case *prom_gce.SDConfig:
diags.AddAll(ValidateDiscoveryGCE(sdc))
case *prom_kubernetes.SDConfig:
diags.AddAll(validateDiscoveryKubernetes(sdc))
case *prom_aws.LightsailSDConfig:
diags.AddAll(validateDiscoveryLightsail(sdc))
default:
diags.Add(diag.SeverityLevelError, fmt.Sprintf("unsupported service discovery %s was provided", serviceDiscoveryConfig.Name()))
}
for _, serviceDiscoveryConfig := range serviceDiscoveryConfigs {
switch sdc := serviceDiscoveryConfig.(type) {
case prom_discover.StaticConfig:
diags.AddAll(validateScrapeTargets(sdc))
case *prom_azure.SDConfig:
diags.AddAll(validateDiscoveryAzure(sdc))
case *prom_consul.SDConfig:
diags.AddAll(validateDiscoveryConsul(sdc))
case *prom_digitalocean.SDConfig:
diags.AddAll(validateDiscoveryDigitalOcean(sdc))
case *prom_dns.SDConfig:
diags.AddAll(validateDiscoveryDns(sdc))
case *prom_docker.DockerSDConfig:
diags.AddAll(validateDiscoveryDocker(sdc))
case *prom_aws.EC2SDConfig:
diags.AddAll(validateDiscoveryEC2(sdc))
case *prom_file.SDConfig:
diags.AddAll(validateDiscoveryFile(sdc))
case *prom_gce.SDConfig:
diags.AddAll(validateDiscoveryGCE(sdc))
case *prom_kubernetes.SDConfig:
diags.AddAll(validateDiscoveryKubernetes(sdc))
case *prom_aws.LightsailSDConfig:
diags.AddAll(validateDiscoveryLightsail(sdc))
default:
diags.Add(diag.SeverityLevelError, fmt.Sprintf("unsupported service discovery %s was provided", serviceDiscoveryConfig.Name()))
}
}

Expand Down
23 changes: 0 additions & 23 deletions converter/internal/promtailconvert/internal/build/azure_sd.go
View file
Open in desktop

This file was deleted.

13 changes: 0 additions & 13 deletions converter/internal/promtailconvert/internal/build/consul_agent.go
View file
Open in desktop

This file was deleted.

Loading