Enable OTel validation mode for OTel input packages in static checks

internal/fields/validate.go

@@ -154,7 +154,7 @@ type Validator struct {
 
 	disabledNormalization bool
 
-	enabledOTELValidation bool
+	enabledOTelValidation bool
 
 	injectFieldsOptions InjectFieldsOptions
 }
@@ -248,10 +248,10 @@ func WithInjectFieldsOptions(options InjectFieldsOptions) ValidatorOption {
 	}
 }
 
-// WithOTELValidation configures the validator to enable or disable OpenTelemetry specific validation.
-func WithOTELValidation(otelValidation bool) ValidatorOption {
+// WithOTelValidation configures the validator to enable or disable OpenTelemetry specific validation.
+func WithOTelValidation(otelValidation bool) ValidatorOption {
 	return func(v *Validator) error {
-		v.enabledOTELValidation = otelValidation
+		v.enabledOTelValidation = otelValidation
 		return nil
 	}
 }
@@ -573,7 +573,7 @@ func (v *Validator) ValidateDocumentMap(body common.MapStr) multierror.Error {
 
 	// If package uses OpenTelemetry Collector, skip field validation and just
 	// validate document values (datasets).
-	if !v.enabledOTELValidation {
+	if !v.enabledOTelValidation {
 		errs = append(errs, v.validateMapElement("", body, body)...)
 	}
 

internal/testrunner/runners/policy/policy.go

@@ -172,7 +172,7 @@ var policyEntryFilters = []policyEntryFilter{
 	}},
 }
 
-var uniqueOtelComponentIDReplace = policyEntryReplace{
+var uniqueOTelComponentIDReplace = policyEntryReplace{
 	regexp:  regexp.MustCompile(`^(\s{2,})([^/]+)/([^:]+):(\s\{\}|\s*)$`),
 	replace: "$1$2/componentid-%s:$4",
 }
@@ -203,10 +203,10 @@ var otelComponentIDsRegexp = regexp.MustCompile(`(?m)^(?:extensions|receivers|pr
 // policies. This preparation is based on removing contents that are generated, or replace them
 // by controlled values.
 func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, error) {
-	// Replacement of the OTEL component IDs needs to be done before unmarshalling the YAML.
-	// The OTEL IDs are keys in maps, and using the policyEntryFilter with memberReplace does
+	// Replacement of the OTel component IDs needs to be done before unmarshalling the YAML.
+	// The OTel IDs are keys in maps, and using the policyEntryFilter with memberReplace does
 	// not ensure to keep the same ordering.
-	policy = replaceOtelComponentIDs(policy)
+	policy = replaceOTelComponentIDs(policy)
 
 	var policyMap common.MapStr
 	err := yaml.Unmarshal(policy, &policyMap)
@@ -222,9 +222,9 @@ func cleanPolicy(policy []byte, entriesToClean []policyEntryFilter) ([]byte, err
 	return yaml.Marshal(policyMap)
 }
 
-// replaceOtelComponentIDs finds OTel Collector component IDs in the policy and replaces them with controlled values.
+// replaceOTelComponentIDs finds OTel Collector component IDs in the policy and replaces them with controlled values.
 // It also replaces references to those IDs in service.extensions and service.pipelines.
-func replaceOtelComponentIDs(policy []byte) []byte {
+func replaceOTelComponentIDs(policy []byte) []byte {
 	replacementsDone := map[string]string{}
 
 	policy = otelComponentIDsRegexp.ReplaceAllFunc(policy, func(match []byte) []byte {
@@ -233,16 +233,16 @@ func replaceOtelComponentIDs(policy []byte) []byte {
 		var section strings.Builder
 		for scanner.Scan() {
 			line := scanner.Text()
-			if uniqueOtelComponentIDReplace.regexp.MatchString(line) {
-				originalOtelID, _, _ := strings.Cut(strings.TrimSpace(line), ":")
+			if uniqueOTelComponentIDReplace.regexp.MatchString(line) {
+				originalOTelID, _, _ := strings.Cut(strings.TrimSpace(line), ":")
 
-				replacement := fmt.Sprintf(uniqueOtelComponentIDReplace.replace, strconv.Itoa(count))
+				replacement := fmt.Sprintf(uniqueOTelComponentIDReplace.replace, strconv.Itoa(count))
 				count++
-				line = uniqueOtelComponentIDReplace.regexp.ReplaceAllString(line, replacement)
+				line = uniqueOTelComponentIDReplace.regexp.ReplaceAllString(line, replacement)
 
 				// store the otel ID replaced without the space indentation and the colon to be replaced later
 				// (e.g. http_check/4391d954-1ffe-4014-a256-5eda78a71828 replaced by http_check/componentid-0)
-				replacementsDone[originalOtelID], _, _ = strings.Cut(strings.TrimSpace(string(line)), ":")
+				replacementsDone[originalOTelID], _, _ = strings.Cut(strings.TrimSpace(string(line)), ":")
 			}
 			section.WriteString(line + "\n")
 		}

internal/testrunner/runners/static/tester.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"slices"
 
 	"github.com/elastic/elastic-package/internal/benchrunner/runners/stream"
 	"github.com/elastic/elastic-package/internal/fields"
@@ -166,6 +167,7 @@ func (r tester) verifySampleEvent(pkgManifest *packages.PackageManifest) []testr
 		fields.WithDefaultNumericConversion(),
 		fields.WithExpectedDatasets(expectedDatasets),
 		fields.WithEnabledImportAllECSSChema(true),
+		fields.WithOTelValidation(isTestUsingOTelCollectorInput(pkgManifest)),
 	)
 	if err != nil {
 		results, _ := resultComposer.WithError(fmt.Errorf("creating fields validator for data stream failed: %w", err))
@@ -234,3 +236,19 @@ func (r tester) getExpectedDatasets(pkgManifest *packages.PackageManifest) ([]st
 func (r tester) TearDown(ctx context.Context) error {
 	return nil // it's a static test runner, no state is stored
 }
+
+func isTestUsingOTelCollectorInput(manifest *packages.PackageManifest) bool {
+	if manifest.Type != "input" {
+		return false
+	}
+
+	// We are not testing an specific policy template here, assume this is an OTel package
+	// if at least one policy template has an "otelcol" input.
+	if !slices.ContainsFunc(manifest.PolicyTemplates, func(t packages.PolicyTemplate) bool {
+		return t.Input == "otelcol"
+	}) {
+		return false
+	}
+
+	return true
+}

internal/testrunner/runners/system/tester.go

@@ -1629,7 +1629,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
 		return nil, err
 	}
 
-	if r.isTestUsingOTELCollectorInput(scenario.policyTemplateInput) {
+	if r.isTestUsingOTelCollectorInput(scenario.policyTemplateInput) {
 		logger.Warn("Validation for packages using OpenTelemetry Collector input is experimental")
 	}
 
@@ -1640,8 +1640,8 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
 		fields.WithExpectedDatasets(expectedDatasets),
 		fields.WithEnabledImportAllECSSChema(true),
 		fields.WithDisableNormalization(scenario.syntheticEnabled),
-		// When using the OTEL collector input, just a subset of validations are performed (e.g. check expected datasets)
-		fields.WithOTELValidation(r.isTestUsingOTELCollectorInput(scenario.policyTemplateInput)),
+		// When using the OTel collector input, just a subset of validations are performed (e.g. check expected datasets)
+		fields.WithOTelValidation(r.isTestUsingOTelCollectorInput(scenario.policyTemplateInput)),
 	)
 	if err != nil {
 		return result.WithErrorf("creating fields validator for data stream failed (path: %s): %w", r.dataStreamPath, err)
@@ -1654,7 +1654,7 @@ func (r *tester) validateTestScenario(ctx context.Context, result *testrunner.Re
 		})
 	}
 
-	if !r.isTestUsingOTELCollectorInput(scenario.policyTemplateInput) && r.fieldValidationMethod == mappingsMethod {
+	if !r.isTestUsingOTelCollectorInput(scenario.policyTemplateInput) && r.fieldValidationMethod == mappingsMethod {
 		logger.Debug("Performing validation based on mappings")
 		exceptionFields := listExceptionFields(scenario.docs, fieldsValidator)
 
@@ -1830,7 +1830,7 @@ func (r *tester) runTest(ctx context.Context, config *testConfig, stackConfig st
 	return r.validateTestScenario(ctx, result, scenario, config)
 }
 
-func (r *tester) isTestUsingOTELCollectorInput(policyTemplateInput string) bool {
+func (r *tester) isTestUsingOTelCollectorInput(policyTemplateInput string) bool {
 	// Just supported for input packages currently
 	if r.pkgManifest.Type != "input" {
 		return false
@@ -2232,8 +2232,8 @@ func (r *tester) checkTransforms(ctx context.Context, config *testConfig, pkgMan
 			fields.WithNumericKeywordFields(config.NumericKeywordFields),
 			fields.WithEnabledImportAllECSSChema(true),
 			fields.WithDisableNormalization(syntheticEnabled),
-			// When using the OTEL collector input, just a subset of validations are performed (e.g. check expected datasets)
-			fields.WithOTELValidation(r.isTestUsingOTELCollectorInput(policyTemplateInput)),
+			// When using the OTel collector input, just a subset of validations are performed (e.g. check expected datasets)
+			fields.WithOTelValidation(r.isTestUsingOTelCollectorInput(policyTemplateInput)),
 		)
 		if err != nil {
 			return fmt.Errorf("creating fields validator for data stream failed (path: %s): %w", transformRootPath, err)

test/packages/parallel/httpcheck/sample_event.json

@@ -0,0 +1,38 @@
+{
+    "@timestamp": "2025-10-28T19:45:38.860Z",
+    "_metric_names_hash": "dbc9255cf6baf03b",
+    "attributes": {
+        "http": {
+            "method": "GET",
+            "status_class": "1xx",
+            "url": "http://svc-web:80"
+        }
+    },
+    "data_stream": {
+        "dataset": "httpcheck.check.otel",
+        "namespace": "18445",
+        "type": "metrics"
+    },
+    "event": {
+        "dataset": "httpcheck.check.otel"
+    },
+    "http": {
+        "method": "GET",
+        "status_class": "1xx",
+        "url": "http://svc-web:80"
+    },
+    "httpcheck": {
+        "status": 0
+    },
+    "metrics": {
+        "httpcheck": {
+            "status": 0
+        }
+    },
+    "scope": {
+        "name": "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/httpcheckreceiver",
+        "version": "9.2.0"
+    },
+    "start_timestamp": "2025-10-28T19:45:37.831Z",
+    "unit": "1"
+}

test/packages/parallel/otel_http_server/docs/README.md

@@ -1,4 +1,4 @@
-# HTTP Server following OTEL Semconv
+# HTTP Server following OTel Semconv
 
 This package includes dashboards for data collected from HTTP Servers following
-OTEL Semconv.
+OTel Semconv.

test/packages/parallel/otel_http_server/manifest.yml

@@ -1,10 +1,10 @@
 format_version: 3.4.0
 name: otel_http_server
-title: "HTTP Server following OTEL Semconv"
+title: "HTTP Server following OTel Semconv"
 version: 0.0.1
 source:
   license: "Elastic-2.0"
-description: "This package provides dashboards for HTTP server metrics that follow OTEL Semantic Conventions."
+description: "This package provides dashboards for HTTP server metrics that follow OTel Semantic Conventions."
 type: content
 categories:
   - web