Skip to content

Commit

Permalink
Refactor prometheusreceiver config to avoid some custom unmarshal work
Browse files Browse the repository at this point in the history 
Signed-off-by: Bogdan Drutu <bogdandrutu@gmail.com>
  • Loading branch information
@bogdandrutu
bogdandrutu committed Dec 15, 2023
1 parent 0af7fce commit b5d929d
Show file tree
Hide file tree
Showing 15 changed files with 175 additions and 223 deletions.
22 changes: 22 additions & 0 deletions .chloggen/refactorprom.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Use this changelog template to create an entry for release notes.

# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
change_type: 'breaking'

# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
component: prometheusreceiver

# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
note: Consolidate Config members and remove the need of placeholders.

# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
issues: [29901]

# If your change doesn't affect end users or the exported elements of any package,
# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
# Optional: The change log or logs in which this entry should be included.
# e.g. '[user]' or '[user, api]'
# Include 'user' if the change is relevant to end users.
# Include 'api' if there is a change to a library API.
# Default: '[user]'
change_logs: [api]
194 changes: 68 additions & 126 deletions receiver/prometheusreceiver/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,25 +16,14 @@ import (
promconfig "github.com/prometheus/prometheus/config"
promHTTP "github.com/prometheus/prometheus/discovery/http"
"github.com/prometheus/prometheus/discovery/kubernetes"
"go.opentelemetry.io/collector/component"
"go.opentelemetry.io/collector/confmap"
"go.uber.org/zap"
"gopkg.in/yaml.v2"
)

const (
// The key for Prometheus scraping configs.
prometheusConfigKey = "config"

// keys to access the http_sd_config from config root
targetAllocatorConfigKey = "target_allocator"
targetAllocatorHTTPSDConfigKey = "http_sd_config"
)

// Config defines configuration for Prometheus receiver.
type Config struct {
PrometheusConfig *promconfig.Config `mapstructure:"-"`
TrimMetricSuffixes bool `mapstructure:"trim_metric_suffixes"`
PrometheusConfig *PrometheusConfig `mapstructure:"config"`
TrimMetricSuffixes bool `mapstructure:"trim_metric_suffixes"`
// UseStartTimeMetric enables retrieving the start time of all counter metrics
// from the process_start_time_seconds metric. This is only correct if all counters on that endpoint
// started after the process start time, and the process is the only actor exporting the metric after
Expand All @@ -47,93 +36,77 @@ type Config struct {
// ReportExtraScrapeMetrics - enables reporting of additional metrics for Prometheus client like scrape_body_size_bytes
ReportExtraScrapeMetrics bool `mapstructure:"report_extra_scrape_metrics"`

TargetAllocator *targetAllocator `mapstructure:"target_allocator"`

// ConfigPlaceholder is just an entry to make the configuration pass a check
// that requires that all keys present in the config actually exist on the
// structure, ie.: it will error if an unknown key is present.
ConfigPlaceholder any `mapstructure:"config"`
TargetAllocator *TargetAllocator `mapstructure:"target_allocator"`

// EnableProtobufNegotiation allows the collector to set the scraper option for
// protobuf negotiation when conferring with a prometheus client.
EnableProtobufNegotiation bool `mapstructure:"enable_protobuf_negotiation"`
}

type targetAllocator struct {
// Validate checks the receiver configuration is valid.
func (cfg *Config) Validate() error {
if cfg.PrometheusConfig != nil && len(cfg.PrometheusConfig.ScrapeConfigs) == 0 && cfg.TargetAllocator == nil {
return errors.New("no Prometheus scrape_configs or target_allocator set")
}
return nil
}

type TargetAllocator struct {
Endpoint string `mapstructure:"endpoint"`
Interval time.Duration `mapstructure:"interval"`
CollectorID string `mapstructure:"collector_id"`
// ConfigPlaceholder is just an entry to make the configuration pass a check
// that requires that all keys present in the config actually exist on the
// structure, ie.: it will error if an unknown key is present.
ConfigPlaceholder any `mapstructure:"http_sd_config"`
HTTPSDConfig *promHTTP.SDConfig `mapstructure:"-"`
HTTPSDConfig *HTTPSDConfig `mapstructure:"http_sd_config"`
}

var _ component.Config = (*Config)(nil)
var _ confmap.Unmarshaler = (*Config)(nil)

func checkFile(fn string) error {
// Nothing set, nothing to error on.
if fn == "" {
func (cfg *TargetAllocator) Validate() error {
// validate targetAllocator
if cfg == nil {
return nil
}
_, err := os.Stat(fn)
return err
}

func checkTLSConfig(tlsConfig commonconfig.TLSConfig) error {
if err := checkFile(tlsConfig.CertFile); err != nil {
return fmt.Errorf("error checking client cert file %q: %w", tlsConfig.CertFile, err)
// ensure valid endpoint
if _, err := url.ParseRequestURI(cfg.Endpoint); err != nil {
return fmt.Errorf("TargetAllocator endpoint is not valid: %s", cfg.Endpoint)
}
if err := checkFile(tlsConfig.KeyFile); err != nil {
return fmt.Errorf("error checking client key file %q: %w", tlsConfig.KeyFile, err)
// ensure valid collectorID without variables
if cfg.CollectorID == "" || strings.Contains(cfg.CollectorID, "${") {
return fmt.Errorf("CollectorID is not a valid ID")
}

return nil
}

// Validate checks the receiver configuration is valid.
func (cfg *Config) Validate() error {
promConfig := cfg.PrometheusConfig
if promConfig != nil {
err := cfg.validatePromConfig(promConfig)
if err != nil {
return err
}
}
// PrometheusConfig is a redeclaration of the promconfig.Config because we need custom unmarshaling
// because prometheus "config" uses `yaml` tags.
type PrometheusConfig promconfig.Config

if cfg.TargetAllocator != nil {
err := cfg.validateTargetAllocatorConfig()
if err != nil {
return err
}
}
return nil
}
var _ confmap.Unmarshaler = (*PrometheusConfig)(nil)

func (cfg *Config) validatePromConfig(promConfig *promconfig.Config) error {
if len(promConfig.ScrapeConfigs) == 0 && cfg.TargetAllocator == nil {
return errors.New("no Prometheus scrape_configs or target_allocator set")
}
func (cfg *PrometheusConfig) Unmarshal(componentParser *confmap.Conf) error {
return unmarshalYamlStruct(componentParser.ToStringMap(), (*promconfig.Config)(cfg))
}

func (cfg *PrometheusConfig) Validate() error {
// Reject features that Prometheus supports but that the receiver doesn't support:
// See:
// * https://github.com/open-telemetry/opentelemetry-collector/issues/3863
// * https://github.com/open-telemetry/wg-prometheus/issues/3
unsupportedFeatures := make([]string, 0, 4)
if len(promConfig.RemoteWriteConfigs) != 0 {
if len(cfg.RemoteWriteConfigs) != 0 {
unsupportedFeatures = append(unsupportedFeatures, "remote_write")
}
if len(promConfig.RemoteReadConfigs) != 0 {
if len(cfg.RemoteReadConfigs) != 0 {
unsupportedFeatures = append(unsupportedFeatures, "remote_read")
}
if len(promConfig.RuleFiles) != 0 {
if len(cfg.RuleFiles) != 0 {
unsupportedFeatures = append(unsupportedFeatures, "rule_files")
}
if len(promConfig.AlertingConfig.AlertRelabelConfigs) != 0 {
if len(cfg.AlertingConfig.AlertRelabelConfigs) != 0 {
unsupportedFeatures = append(unsupportedFeatures, "alert_config.relabel_configs")
}
if len(promConfig.AlertingConfig.AlertmanagerConfigs) != 0 {
if len(cfg.AlertingConfig.AlertmanagerConfigs) != 0 {
unsupportedFeatures = append(unsupportedFeatures, "alert_config.alertmanagers")
}
if len(unsupportedFeatures) != 0 {
Expand All @@ -142,7 +115,7 @@ func (cfg *Config) validatePromConfig(promConfig *promconfig.Config) error {
return fmt.Errorf("unsupported features:\n\t%s", strings.Join(unsupportedFeatures, "\n\t"))
}

for _, sc := range cfg.PrometheusConfig.ScrapeConfigs {
for _, sc := range cfg.ScrapeConfigs {
if sc.HTTPClientConfig.Authorization != nil {
if err := checkFile(sc.HTTPClientConfig.Authorization.CredentialsFile); err != nil {
return fmt.Errorf("error checking authorization credentials file %q: %w", sc.HTTPClientConfig.Authorization.CredentialsFile, err)
Expand All @@ -164,84 +137,53 @@ func (cfg *Config) validatePromConfig(promConfig *promconfig.Config) error {
return nil
}

func (cfg *Config) validateTargetAllocatorConfig() error {
// validate targetAllocator
targetAllocatorConfig := cfg.TargetAllocator
if targetAllocatorConfig == nil {
// HTTPSDConfig is a redeclaration of the promHTTP.SDConfig because we need custom unmarshaling
// because prometheus "config" uses `yaml` tags.
type HTTPSDConfig promHTTP.SDConfig

var _ confmap.Unmarshaler = (*HTTPSDConfig)(nil)

func (cfg *HTTPSDConfig) Unmarshal(componentParser *confmap.Conf) error {
sdConfigMap := componentParser.ToStringMap()
if len(sdConfigMap) == 0 {
return nil
}
// ensure valid endpoint
if _, err := url.ParseRequestURI(targetAllocatorConfig.Endpoint); err != nil {
return fmt.Errorf("TargetAllocator endpoint is not valid: %s", targetAllocatorConfig.Endpoint)
}
// ensure valid collectorID without variables
if targetAllocatorConfig.CollectorID == "" || strings.Contains(targetAllocatorConfig.CollectorID, "${") {
return fmt.Errorf("CollectorID is not a valid ID")
}

return nil
sdConfigMap["url"] = "http://placeholder" // we have to set it as else the marshal will fail
return unmarshalYamlStruct(sdConfigMap, (*promHTTP.SDConfig)(cfg))
}

// Unmarshal a config.Parser into the config struct.
func (cfg *Config) Unmarshal(componentParser *confmap.Conf) error {
if componentParser == nil {
func unmarshalYamlStruct(in map[string]any, out any) error {
if len(in) == 0 {
return nil
}
// We need custom unmarshaling because prometheus "config" subkey defines its own
// YAML unmarshaling routines so we need to do it explicitly.

err := componentParser.Unmarshal(cfg, confmap.WithErrorUnused())
yamlOut, err := yaml.Marshal(in)
if err != nil {
return fmt.Errorf("prometheus receiver failed to parse config: %w", err)
return fmt.Errorf("prometheus receiver: failed to marshal config to yaml: %w", err)
}

// Unmarshal prometheus's config values. Since prometheus uses `yaml` tags, so use `yaml`.
promCfg, err := componentParser.Sub(prometheusConfigKey)
if err != nil || len(promCfg.ToStringMap()) == 0 {
return err
}
out, err := yaml.Marshal(promCfg.ToStringMap())
err = yaml.UnmarshalStrict(yamlOut, out)
if err != nil {
return fmt.Errorf("prometheus receiver failed to marshal config to yaml: %w", err)
return fmt.Errorf("prometheus receiver: failed to unmarshal yaml to prometheus config object: %w", err)
}
return nil
}

err = yaml.UnmarshalStrict(out, &cfg.PrometheusConfig)
if err != nil {
return fmt.Errorf("prometheus receiver failed to unmarshal yaml to prometheus config: %w", err)
func checkFile(fn string) error {
// Nothing set, nothing to error on.
if fn == "" {
return nil
}
_, err := os.Stat(fn)
return err
}

// Unmarshal targetAllocator configs
targetAllocatorCfg, err := componentParser.Sub(targetAllocatorConfigKey)
if err != nil {
return err
}
targetAllocatorHTTPSDCfg, err := targetAllocatorCfg.Sub(targetAllocatorHTTPSDConfigKey)
if err != nil {
return err
func checkTLSConfig(tlsConfig commonconfig.TLSConfig) error {
if err := checkFile(tlsConfig.CertFile); err != nil {
return fmt.Errorf("error checking client cert file %q: %w", tlsConfig.CertFile, err)
}

targetAllocatorHTTPSDMap := targetAllocatorHTTPSDCfg.ToStringMap()
if len(targetAllocatorHTTPSDMap) != 0 {
targetAllocatorHTTPSDMap["url"] = "http://placeholder" // we have to set it as else the marshal will fail
httpSDConf, err := yaml.Marshal(targetAllocatorHTTPSDMap)
if err != nil {
return fmt.Errorf("prometheus receiver failed to marshal config to yaml: %w", err)
}
err = yaml.UnmarshalStrict(httpSDConf, &cfg.TargetAllocator.HTTPSDConfig)
if err != nil {
return fmt.Errorf("prometheus receiver failed to unmarshal yaml to prometheus config: %w", err)
}
if err := checkFile(tlsConfig.KeyFile); err != nil {
return fmt.Errorf("error checking client key file %q: %w", tlsConfig.KeyFile, err)
}

return nil
}

func configWarnings(logger *zap.Logger, cfg *Config) {
for _, sc := range cfg.PrometheusConfig.ScrapeConfigs {
for _, rc := range sc.MetricRelabelConfigs {
if rc.TargetLabel == "__name__" {
logger.Warn("metric renaming using metric_relabel_configs will result in unknown-typed metrics without a unit or description", zap.String("job", sc.JobName))
}
}
}
}

0 comments on commit b5d929d

Please sign in to comment.