Validate fields using ECS JSON schema when all stack versions support ecs@mappings

internal/builder/dynamic_mappings.go

@@ -75,7 +75,7 @@ func addDynamicMappings(packageRoot, destinationDir string) error {
 		if err != nil {
 			return err
 		}
-		os.WriteFile(packageManifest, contents, 0664)
+		err = os.WriteFile(packageManifest, contents, 0664)
 		if err != nil {
 			return err
 		}

internal/fields/dependency_manager.go

@@ -302,7 +302,7 @@ func (dm *DependencyManager) importField(schemaName, fieldPath string) (FieldDef
 	return *imported, nil
 }
 
-// ImportAllFields method resolves all fields avaialble in the default ECS schema.
+// ImportAllFields method resolves all fields available in the default ECS schema.
 func (dm *DependencyManager) ImportAllFields(schemaName string) ([]FieldDefinition, error) {
 	if dm == nil {
 		return []FieldDefinition{}, fmt.Errorf(`importing all external fields: external fields not allowed because dependencies file "_dev/build/build.yml" is missing`)

internal/fields/validate.go

@@ -34,6 +34,92 @@ var (
 	semver2_3_0 = semver.MustParse("2.3.0")
 	semver3_0_1 = semver.MustParse("3.0.1")
 
+	// List of stack releases that do not
+	// include ECS mappings (all versions before 8.13.0).
+	stackVersionsWithoutECS = []*semver.Version{
+		semver.MustParse("8.12.2"),
+		semver.MustParse("8.12.1"),
+		semver.MustParse("8.12.0"),
+		semver.MustParse("8.11.4"),
+		semver.MustParse("8.11.3"),
+		semver.MustParse("8.11.2"),
+		semver.MustParse("8.11.1"),
+		semver.MustParse("8.11.0"),
+		semver.MustParse("8.10.4"),
+		semver.MustParse("8.10.3"),
+		semver.MustParse("8.10.2"),
+		semver.MustParse("8.10.1"),
+		semver.MustParse("8.10.0"),
+		semver.MustParse("8.9.2"),
+		semver.MustParse("8.9.1"),
+		semver.MustParse("8.9.0"),
+		semver.MustParse("8.8.2"),
+		semver.MustParse("8.8.1"),
+		semver.MustParse("8.8.0"),
+		semver.MustParse("8.7.1"),
+		semver.MustParse("8.7.0"),
+		semver.MustParse("8.6.2"),
+		semver.MustParse("8.6.1"),
+		semver.MustParse("8.6.0"),
+		semver.MustParse("8.5.3"),
+		semver.MustParse("8.5.2"),
+		semver.MustParse("8.5.1"),
+		semver.MustParse("8.5.0"),
+		semver.MustParse("8.4.3"),
+		semver.MustParse("8.4.2"),
+		semver.MustParse("8.4.1"),
+		semver.MustParse("8.4.0"),
+		semver.MustParse("8.3.3"),
+		semver.MustParse("8.3.2"),
+		semver.MustParse("8.3.1"),
+		semver.MustParse("8.3.0"),
+		semver.MustParse("8.2.3"),
+		semver.MustParse("8.2.2"),
+		semver.MustParse("8.2.1"),
+		semver.MustParse("8.2.0"),
+		semver.MustParse("8.1.3"),
+		semver.MustParse("8.1.2"),
+		semver.MustParse("8.1.1"),
+		semver.MustParse("8.1.0"),
+		semver.MustParse("8.0.1"),
+		semver.MustParse("8.0.0"),
+		semver.MustParse("7.17.24"),
+		semver.MustParse("7.17.23"),
+		semver.MustParse("7.17.22"),
+		semver.MustParse("7.17.21"),
+		semver.MustParse("7.17.20"),
+		semver.MustParse("7.17.19"),
+		semver.MustParse("7.17.18"),
+		semver.MustParse("7.17.17"),
+		semver.MustParse("7.17.16"),
+		semver.MustParse("7.17.15"),
+		semver.MustParse("7.17.14"),
+		semver.MustParse("7.17.13"),
+		semver.MustParse("7.17.12"),
+		semver.MustParse("7.17.11"),
+		semver.MustParse("7.17.10"),
+		semver.MustParse("7.17.9"),
+		semver.MustParse("7.17.8"),
+		semver.MustParse("7.17.7"),
+		semver.MustParse("7.17.6"),
+		semver.MustParse("7.17.5"),
+		semver.MustParse("7.17.4"),
+		semver.MustParse("7.17.3"),
+		semver.MustParse("7.17.2"),
+		semver.MustParse("7.17.1"),
+		semver.MustParse("7.17.0"),
+		semver.MustParse("7.16.3"),
+		semver.MustParse("7.16.2"),
+		semver.MustParse("7.16.1"),
+		semver.MustParse("7.16.0"),
+		semver.MustParse("7.15.2"),
+		semver.MustParse("7.15.1"),
+		semver.MustParse("7.15.0"),
+		semver.MustParse("7.14.2"),
+		semver.MustParse("7.14.1"),
+		semver.MustParse("7.14.0"), // First version of Fleet in GA; there are no packages older than this version.
+	}
+
 	defaultExternal = "ecs"
 )
 
@@ -215,18 +301,91 @@ func initDependencyManagement(packageRoot string, specVersion semver.Version, im
 		return nil, nil, fmt.Errorf("can't create field dependency manager: %w", err)
 	}
 
+	// Check if the package embeds ECS mappings
+	packageEmbedsEcsMappings := buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0)
+	if !packageEmbedsEcsMappings {
+		logger.Debugf("Package does not embed ECS mappings")
+	}
+
+	// Check if all stack versions support ECS mappings
+	stackSupportsEcsMapping, err := supportsECSMappings(packageRoot)
+	if err != nil {
+		return nil, nil, fmt.Errorf("can't check if stack version includes ECS mappings: %w", err)
+	}
+
+	// If the package embeds ECS mappings, or the stack version includes ECS mappings, then
+	// we should import the ECS schema to validate the package fields against it.
 	var schema []FieldDefinition
-	if buildManifest.ImportMappings() && !specVersion.LessThan(semver2_3_0) && importECSSchema {
+	if (packageEmbedsEcsMappings || stackSupportsEcsMapping) && importECSSchema {
+		// Import all fields from external schema (most likely ECS) to
+		// validate the package fields against it.
 		ecsSchema, err := fdm.ImportAllFields(defaultExternal)
 		if err != nil {
 			return nil, nil, err
 		}
+		logger.Debug("Imported ECS fields definition from external schema for validation")
 		schema = ecsSchema
 	}
 
 	return fdm, schema, nil
 }
 
+// supportsECSMappings check if all the versions of the stack the package can run on support ECS mappings.
+func supportsECSMappings(packageRoot string) (bool, error) {
+	packageManifest, err := packages.ReadPackageManifestFromPackageRoot(packageRoot)
+	if err != nil {
+		return false, fmt.Errorf("can't read package manifest: %w", err)
+	}
+
+	if len(packageManifest.Conditions.Kibana.Version) == 0 {
+		logger.Debugf("No Kibana version constraint found in package manifest; assuming it does not support ECS mappings.")
+		return false, nil
+	}
+
+	kibanaConstraints, err := semver.NewConstraint(packageManifest.Conditions.Kibana.Version)
+	if err != nil {
+		return false, fmt.Errorf("invalid constraint for Kibana: %w", err)
+	}
+
+	return allVersionsIncludeECS(kibanaConstraints), nil
+}
+
+// allVersionsIncludeECS Check if all the stack versions in the constraints include ECS mappings. Only the stack
+// versions 8.13.0 and above include ECS mappings.
+//
+// Returns true if all the stack versions in the constraints include ECS mappings, otherwise returns false.
+func allVersionsIncludeECS(kibanaConstraints *semver.Constraints) bool {
+	// Looking for a version that satisfies the package constraints.
+	for _, v := range stackVersionsWithoutECS {
+		if kibanaConstraints.Check(v) {
+			// Found a version that satisfies the constraints,
+			// so at least this version does not include
+			// ECS mappings.
+			return false
+		}
+	}
+
+	// If no version satisfies the constraints, then all versions
+	// include ECS mappings.
+	return true
+
+	// This check works under the assumption the constraints are not limited
+	// upwards.
+	//
+	// For example, if the constraint is `>= 8.12.0` and the stack version is
+	// `8.12.999`, the constraint will be satisfied.
+	//
+	// However, if the constraint is `>= 8.0.0, < 8.10.0` the check will not
+	// return the right result.
+	//
+	// To support this, we would need to check the constraint against a larger
+	// set of versions, and check if the constraint is satisfied for all
+	// of them, like in the commented out example above.
+	//
+	// lastStackVersionWithoutEcsMappings := semver.MustParse("8.12.999")
+	// return !kibanaConstraints.Check(lastStackVersionWithoutEcsMappings)
+}
+
 //go:embed _static/allowed_geo_ips.txt
 var allowedGeoIPs string
 

internal/fields/validate_test.go

@@ -961,6 +961,58 @@ func TestValidateExternalMultiField(t *testing.T) {
 	require.Empty(t, errs)
 }
 
+func TestValidateStackVersionsWithEcsMappings(t *testing.T) {
+	// List of unique stack constraints extracted from the
+	// package manifest files in the elastic/integrations
+	// repository.
+	constraints := []struct {
+		Constraints string
+		SupportEcs  bool
+	}{
+		{"^7.17.0", false},
+		{"7.17.19 || > 8.13", false},
+		{"^7.14.0 || ^8.0.0", false},
+		{"^7.14.1 || ^8.0.0", false},
+		{"^7.14.1 || ^8.8.0", false},
+		{"^7.16.0 || ^8.0.0", false},
+		{"^7.17.0 || ^8.0.0", false},
+		{"^8.0.0", false},
+		{"^8.10.1", false},
+		{"^8.10.2", false},
+		{"^8.11.0", false},
+		{"^8.11.2", false},
+		{"^8.12.0", false},
+		{"^8.12.1", false},
+		{"^8.12.2", false},
+		{"^8.13.0", true},
+		{"^8.14.0", true},
+		{"^8.2.0", false},
+		{"^8.2.1", false},
+		{"^8.3.0", false},
+		{"^8.4.0", false},
+		{"^8.5.0", false},
+		{"^8.5.1", false},
+		{"^8.6.0", false},
+		{"^8.6.1", false},
+		{"^8.7.0", false},
+		{"^8.7.1", false},
+		{"^8.8.0", false},
+		{"^8.8.1", false},
+		{"^8.8.2", false},
+		{"^8.9.0", false},
+		{">= 8.0.0, < 8.10.0", false},
+		{">= 8.0.0, < 8.0.1", false},
+	}
+
+	for _, c := range constraints {
+		constraint, err := semver.NewConstraint(c.Constraints)
+		if err != nil {
+			require.NoError(t, err)
+		}
+		assert.Equal(t, c.SupportEcs, allVersionsIncludeECS(constraint), "constraint: %s", c.Constraints)
+	}
+}
+
 func readTestResults(t *testing.T, path string) (f results) {
 	c, err := os.ReadFile(path)
 	require.NoError(t, err)

internal/packages/buildmanifest/build_manifest.go

@@ -35,7 +35,7 @@ func (bm *BuildManifest) HasDependencies() bool {
 	return bm.Dependencies.ECS.Reference != ""
 }
 
-// HasDependencies function checks if there are any dependencies defined.
+// ImportMappings function checks if there are any dependencies defined.
 func (bm *BuildManifest) ImportMappings() bool {
 	return bm.Dependencies.ECS.ImportMappings
 }

test/packages/other/ecs_mappings/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@v8.11.0

test/packages/other/ecs_mappings/_dev/build/docs/README.md

@@ -0,0 +1,14 @@
+# ECS Mappings Test
+
+Test package to verify support for ECS mappings available to integrations running on stack version 8.13.0 and later.
+
+Please note that the package:
+
+- does not embed the legacy ECS mappings (no `import_mappings`).
+- does not define fields in the `ecs.yml` file. 
+
+Mappings for ECS fields (for example, `ecs.version`) come from the `ecs@mappings` component template in the integration index template, which has been available since 8.13.0. 
+
+{{event "first"}}
+
+{{fields "first"}}

test/packages/other/ecs_mappings/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

test/packages/other/ecs_mappings/data_stream/first/agent/stream/stream.yml.hbs

@@ -0,0 +1,7 @@
+paths:
+{{#each paths as |path i|}}
+  - {{path}}
+{{/each}}
+exclude_files: [".gz$"]
+processors:
+  - add_locale: ~

test/packages/other/ecs_mappings/data_stream/first/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,10 @@
+---
+description: Pipeline for processing sample logs
+processors:
+- set:
+    field: sample_field
+    value: "1"
+on_failure:
+- set:
+    field: error.message
+    value: '{{ _ingest.on_failure_message }}'

test/packages/other/ecs_mappings/data_stream/first/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/other/ecs_mappings/data_stream/first/fields/histogram-fields.yml

@@ -0,0 +1,3 @@
+- name: service.status.*.histogram
+  type: object
+  object_type: histogram

test/packages/other/ecs_mappings/data_stream/first/manifest.yml

@@ -0,0 +1,13 @@
+title: "First"
+type: logs
+streams:
+  - input: logfile
+    title: Sample logs
+    description: Collect sample logs
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        default:
+          - /var/log/*.log

test/packages/other/ecs_mappings/data_stream/first/sample_event.json

@@ -0,0 +1,29 @@
+{
+    "source.geo.location": {
+        "lat": 1.0,
+        "lon": "2.0"
+    },
+    "destination.geo.location.lat": 3.0,
+    "destination.geo.location.lon": 4.0,
+    "service.status.duration.histogram": {
+        "counts": [
+            8,
+            17,
+            8,
+            7,
+            6,
+            2
+        ],
+        "values": [
+            0.1,
+            0.25,
+            0.35,
+            0.4,
+            0.45,
+            0.5
+        ]
+    },
+    "ecs": {
+        "version": "8.11.0"
+    }
+}