[cmd/builder] Allow configuring confmap providers (#9513)

**Description:**

Allow configuring confmap providers in the builder's config. If the
field isn't set, the default set of providers is used.

**Link to tracking Issue:**

Resolves
#4759.

**Testing:**

Extended unit tests.

**Documentation:**

Updated the readme to include the new options in the example manifest
file.

cc @mx-psi

---------

Co-authored-by: Evan Bradley <evan-bradley@users.noreply.github.com>
Co-authored-by: Pablo Baeyens <pbaeyens31+github@gmail.com>

.chloggen/builder-confmap-providers.yaml

@@ -0,0 +1,31 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. otlpreceiver)
+component: cmd/builder
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Allow configuring `confmap.Provider`s in the builder.
+
+# One or more tracking issues or pull requests related to the change
+issues: [4759]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+  If no providers are specified, the defaults are used.
+  The default providers are: env, file, http, https, and yaml.
+
+  To configure providers, use the `providers` key in your OCB build
+  manifest with a list of Go modules for your providers.
+  The modules will work the same as other Collector components.
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

cmd/builder/README.md

@@ -12,14 +12,20 @@ dist:
   description: Local OpenTelemetry Collector binary
   output_path: /tmp/dist
 exporters:
-  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.86.0
-  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.86.0
+  - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/alibabacloudlogserviceexporter v0.99.0
+  - gomod: go.opentelemetry.io/collector/exporter/debugexporter v0.99.0
 
 receivers:
-  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.86.0
+  - gomod: go.opentelemetry.io/collector/receiver/otlpreceiver v0.99.0
 
 processors:
-  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.86.0
+  - gomod: go.opentelemetry.io/collector/processor/batchprocessor v0.99.0
+
+providers:
+  - gomod: go.opentelemetry.io/collector/confmap/provider/fileprovider v0.99.0
+
+converters:
+  - gomod: go.opentelemetry.io/collector/confmap/converter/expandconverter v0.99.0
 EOF
 $ builder --config=otelcol-builder.yaml
 $ cat > /tmp/otelcol.yaml <<EOF

cmd/builder/internal/builder/config.go

@@ -23,7 +23,8 @@ var ErrInvalidGoMod = errors.New("invalid gomod specification for module")
 
 // Config holds the builder's configuration
 type Config struct {
-	Logger               *zap.Logger
+	Logger *zap.Logger
+
 	SkipGenerate         bool   `mapstructure:"-"`
 	SkipCompilation      bool   `mapstructure:"-"`
 	SkipGetModules       bool   `mapstructure:"-"`
@@ -37,22 +38,24 @@ type Config struct {
 	Receivers    []Module     `mapstructure:"receivers"`
 	Processors   []Module     `mapstructure:"processors"`
 	Connectors   []Module     `mapstructure:"connectors"`
+	Providers    *[]Module    `mapstructure:"providers"`
 	Replaces     []string     `mapstructure:"replaces"`
 	Excludes     []string     `mapstructure:"excludes"`
 }
 
 // Distribution holds the parameters for the final binary
 type Distribution struct {
-	Module               string `mapstructure:"module"`
-	Name                 string `mapstructure:"name"`
-	Go                   string `mapstructure:"go"`
-	Description          string `mapstructure:"description"`
-	OtelColVersion       string `mapstructure:"otelcol_version"`
-	RequireOtelColModule bool   `mapstructure:"-"` // required for backwards-compatibility with builds older than 0.86.0
-	OutputPath           string `mapstructure:"output_path"`
-	Version              string `mapstructure:"version"`
-	BuildTags            string `mapstructure:"build_tags"`
-	DebugCompilation     bool   `mapstructure:"debug_compilation"`
+	Module                   string `mapstructure:"module"`
+	Name                     string `mapstructure:"name"`
+	Go                       string `mapstructure:"go"`
+	Description              string `mapstructure:"description"`
+	OtelColVersion           string `mapstructure:"otelcol_version"`
+	RequireOtelColModule     bool   `mapstructure:"-"` // required for backwards-compatibility with builds older than 0.86.0
+	SupportsConfmapFactories bool   `mapstructure:"-"` // Required for backwards-compatibility with builds older than 0.99.0
+	OutputPath               string `mapstructure:"output_path"`
+	Version                  string `mapstructure:"version"`
+	BuildTags                string `mapstructure:"build_tags"`
+	DebugCompilation         bool   `mapstructure:"debug_compilation"`
 }
 
 // Module represents a receiver, exporter, processor or extension for the distribution
@@ -87,12 +90,17 @@ func NewDefaultConfig() Config {
 
 // Validate checks whether the current configuration is valid
 func (c *Config) Validate() error {
+	var providersError error
+	if c.Providers != nil {
+		providersError = validateModules(*c.Providers)
+	}
 	return multierr.Combine(
 		validateModules(c.Extensions),
 		validateModules(c.Receivers),
 		validateModules(c.Exporters),
 		validateModules(c.Processors),
 		validateModules(c.Connectors),
+		providersError,
 	)
 }
 
@@ -112,7 +120,8 @@ func (c *Config) SetGoPath() error {
 	return nil
 }
 
-func (c *Config) SetRequireOtelColModule() error {
+func (c *Config) SetBackwardsCompatibility() error {
+	// check whether we need to adjust the core API module import
 	constraint, err := version.NewConstraint(">= 0.86.0")
 	if err != nil {
 		return err
@@ -124,6 +133,20 @@ func (c *Config) SetRequireOtelColModule() error {
 	}
 
 	c.Distribution.RequireOtelColModule = constraint.Check(otelColVersion)
+
+	// check whether confmap factories are supported
+	constraint, err = version.NewConstraint(">= 0.99.0")
+	if err != nil {
+		return err
+	}
+
+	otelColVersion, err = version.NewVersion(c.Distribution.OtelColVersion)
+	if err != nil {
+		return err
+	}
+
+	c.Distribution.SupportsConfmapFactories = constraint.Check(otelColVersion)
+
 	return nil
 }
 
@@ -156,6 +179,36 @@ func (c *Config) ParseModules() error {
 		return err
 	}
 
+	if c.Providers != nil {
+		providers, err := parseModules(*c.Providers)
+		if err != nil {
+			return err
+		}
+		c.Providers = &providers
+	} else {
+		providers, err := parseModules([]Module{
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/envprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/fileprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/httpprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/httpsprovider v" + c.Distribution.OtelColVersion,
+			},
+			{
+				GoMod: "go.opentelemetry.io/collector/confmap/provider/yamlprovider v" + c.Distribution.OtelColVersion,
+			},
+		})
+		if err != nil {
+			return err
+		}
+		c.Providers = &providers
+	}
+
 	return nil
 }
 

cmd/builder/internal/builder/config_test.go

@@ -220,8 +220,89 @@ func TestRequireOtelColModule(t *testing.T) {
 		t.Run(tt.Version, func(t *testing.T) {
 			cfg := NewDefaultConfig()
 			cfg.Distribution.OtelColVersion = tt.Version
-			require.NoError(t, cfg.SetRequireOtelColModule())
+			require.NoError(t, cfg.SetBackwardsCompatibility())
 			assert.Equal(t, tt.ExpectedRequireOtelColModule, cfg.Distribution.RequireOtelColModule)
 		})
 	}
 }
+
+func TestConfmapFactoryVersions(t *testing.T) {
+	testCases := []struct {
+		version   string
+		supported bool
+		err       bool
+	}{
+		{
+			version:   "x.0.0",
+			supported: false,
+			err:       true,
+		},
+		{
+			version:   "0.x.0",
+			supported: false,
+			err:       true,
+		},
+		{
+			version:   "0.0.0",
+			supported: false,
+		},
+		{
+			version:   "0.98.0",
+			supported: false,
+		},
+		{
+			version:   "0.98.1",
+			supported: false,
+		},
+		{
+			version:   "0.99.0",
+			supported: true,
+		},
+		{
+			version:   "0.99.7",
+			supported: true,
+		},
+		{
+			version:   "0.100.0",
+			supported: true,
+		},
+		{
+			version:   "0.100.1",
+			supported: true,
+		},
+		{
+			version:   "1.0",
+			supported: true,
+		},
+		{
+			version:   "1.0.0",
+			supported: true,
+		},
+	}
+
+	for _, tt := range testCases {
+		t.Run(tt.version, func(t *testing.T) {
+			cfg := NewDefaultConfig()
+			cfg.Distribution.OtelColVersion = tt.version
+			if !tt.err {
+				require.NoError(t, cfg.SetBackwardsCompatibility())
+				assert.Equal(t, tt.supported, cfg.Distribution.SupportsConfmapFactories)
+			} else {
+				require.Error(t, cfg.SetBackwardsCompatibility())
+			}
+		})
+	}
+}
+
+func TestAddsDefaultProviders(t *testing.T) {
+	cfg := NewDefaultConfig()
+	cfg.Providers = nil
+	assert.NoError(t, cfg.ParseModules())
+	assert.Len(t, *cfg.Providers, 5)
+}
+
+func TestSkipsNilFieldValidation(t *testing.T) {
+	cfg := NewDefaultConfig()
+	cfg.Providers = nil
+	assert.NoError(t, cfg.Validate())
+}

cmd/builder/internal/builder/main.go

@@ -235,7 +235,8 @@ func (c *Config) allComponents() []Module {
 		append(c.Receivers,
 			append(c.Processors,
 				append(c.Extensions,
-					c.Connectors...)...)...)...)
+					append(c.Connectors,
+						*c.Providers...)...)...)...)...)
 }
 
 func (c *Config) readGoModFile() (string, map[string]string, error) {