Fix template path validation including linked files

code/go/internal/validator/semantic/validate_input_policy_template_template_path.go

@@ -6,9 +6,7 @@ package semantic
 
 import (
 	"errors"
-	"fmt"
 	"io/fs"
-	"os"
 	"path"
 
 	"gopkg.in/yaml.v3"
@@ -85,13 +83,16 @@ func validateInputPackagePolicyTemplate(fsys fspath.FS, policyTemplate inputPoli
 }
 
 func validateAgentInputTemplatePath(fsys fspath.FS, tmplPath string) error {
-	templatePath := path.Join("agent", "input", tmplPath)
-	_, err := fs.Stat(fsys, templatePath)
+	dir := path.Join("agent", "input")
+	foundFile, err := findPathAtDirectory(fsys, dir, tmplPath)
 	if err != nil {
-		if errors.Is(err, os.ErrNotExist) {
+		if errors.Is(err, fs.ErrNotExist) {
 			return errTemplateNotFound
 		}
-		return fmt.Errorf("failed to stat template file %s: %w", fsys.Path(templatePath), err)
+		return err
+	}
+	if foundFile == "" {
+		return errTemplateNotFound
 	}
 
 	return nil

code/go/internal/validator/semantic/validate_integration_policy_template_path.go

@@ -5,10 +5,10 @@
 package semantic
 
 import (
-	"errors"
 	"fmt"
 	"io/fs"
 	"path"
+	"strings"
 
 	"gopkg.in/yaml.v3"
 
@@ -148,23 +148,47 @@ func validateInputWithStreams(fsys fspath.FS, input string, dsMap map[string]dat
 				stream.TemplatePath = defaultStreamTemplatePath
 			}
 
-			_, err := fs.ReadFile(fsys, path.Join(dsDir, "agent", "stream", stream.TemplatePath))
+			dir := path.Join(dsDir, "agent", "stream")
+			foundFile, err := findPathAtDirectory(fsys, dir, stream.TemplatePath)
 			if err != nil {
-				if errors.Is(err, fs.ErrNotExist) {
-					// fallback to glob pattern matching in case the default template path is customized with a prefix
-					matches, err := fs.Glob(fsys, path.Join(dsDir, "agent", "stream", "*"+stream.TemplatePath))
-					if err != nil {
-						return err
-					}
-					if len(matches) == 0 {
-						return errTemplateNotFound
-					}
-					continue
-				}
 				return err
 			}
+			if foundFile == "" {
+				return errTemplateNotFound
+			}
+
+			return nil
 		}
 	}
 
 	return nil
 }
+
+// findPathAtDirectory looks for a file matching the templatePath in the given directory (dir)
+// It checks for exact matches, files ending with the templatePath, or templatePath + ".link"
+func findPathAtDirectory(fsys fspath.FS, dir, templatePath string) (string, error) {
+	// Check for exact match, files ending with stream.TemplatePath, or stream.TemplatePath + ".link"
+	entries, err := fs.ReadDir(fsys, dir)
+	if err != nil {
+		return "", err
+	}
+
+	// Filter matches to ensure they match our criteria:
+	// 1. Exact name match
+	// 2. Ends with stream.TemplatePath
+	// 3. Equals stream.TemplatePath + ".link"
+	var foundFile string
+	for _, entry := range entries {
+		name := entry.Name()
+		if name == templatePath || name == templatePath+".link" {
+			foundFile = path.Join(dir, name)
+			break
+		}
+		// fallback to check for suffix match, in case the path is prefixed
+		if strings.HasSuffix(name, templatePath) || strings.HasSuffix(name, templatePath+".link") {
+			foundFile = path.Join(dir, name)
+			break
+		}
+	}
+	return foundFile, nil
+}

code/go/internal/validator/semantic/validate_integration_policy_template_path_test.go

@@ -179,3 +179,106 @@ streams:
 	errs := ValidateIntegrationPolicyTemplates(fspath.DirFS(d))
 	require.Empty(t, errs)
 }
+func TestFindPathAtDirectory(t *testing.T) {
+	d := t.TempDir()
+
+	dsDir := filepath.ToSlash(path.Join("data_stream", "logs", "agent", "stream"))
+	err := os.MkdirAll(filepath.Join(d, "data_stream", "logs", "agent", "stream"), 0o755)
+	require.NoError(t, err)
+
+	t.Run("exact match", func(t *testing.T) {
+		templatePath := "exact.yml.hbs"
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath), []byte("content"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath))
+
+		foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, templatePath)), foundFile)
+	})
+
+	t.Run("match with .link extension", func(t *testing.T) {
+		templatePath := "linked.yml.hbs"
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath+".link"), []byte("content"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", templatePath+".link"))
+
+		foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, templatePath+".link")), foundFile)
+	})
+
+	t.Run("match with prefix", func(t *testing.T) {
+		templatePath := "stream.yml.hbs"
+		prefixedFile := "prefixstream.yml.hbs"
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile), []byte("content"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile))
+
+		foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, prefixedFile)), foundFile)
+	})
+
+	t.Run("match with prefix and .link extension", func(t *testing.T) {
+		templatePath := "stream.yml.hbs"
+		prefixedFile := "prefixstream.yml.hbs.link"
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile), []byte("content"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile))
+
+		foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, prefixedFile)), foundFile)
+	})
+
+	t.Run("no match found", func(t *testing.T) {
+		templatePath := "nonexistent.yml.hbs"
+
+		foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.Empty(t, foundFile)
+	})
+
+	t.Run("multiple matches - exact match takes precedence", func(t *testing.T) {
+		templatePath := "multi.yml.hbs"
+		exactFile := "multi.yml.hbs"
+		prefixedFile := "prefixmulti.yml.hbs"
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", exactFile), []byte("exact"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", exactFile))
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile), []byte("prefixed"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", prefixedFile))
+
+		foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, exactFile)), foundFile)
+	})
+
+	t.Run("link file takes precedence over suffix match", func(t *testing.T) {
+		templatePath := "link.yml.hbs"
+		linkFile := "link.yml.hbs.link"
+		suffixFile := "prefixlink.yml.hbs"
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", linkFile), []byte("link"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", linkFile))
+
+		err = os.WriteFile(filepath.Join(d, "data_stream", "logs", "agent", "stream", suffixFile), []byte("suffix"), 0o644)
+		require.NoError(t, err)
+		defer os.Remove(filepath.Join(d, "data_stream", "logs", "agent", "stream", suffixFile))
+
+		foundFile, err := findPathAtDirectory(fspath.DirFS(d), dsDir, templatePath)
+		require.NoError(t, err)
+		require.NotEmpty(t, foundFile)
+		require.Equal(t, filepath.ToSlash(path.Join(dsDir, linkFile)), foundFile)
+	})
+}

spec/changelog.yml

@@ -15,6 +15,11 @@
     - description: Require defining agent version constraints in input and integration packages.
       type: breaking-change
       link: https://github.com/elastic/package-spec/pull/999
+- version: 3.5.2
+  changes:
+    - description: Fix validation of policy_template paths with linked files.
+      type: bugfix
+      link: https://github.com/elastic/package-spec/pull/1009
 - version: 3.5.1
   changes:
     - description: Input packages don't require to define fields.

test/packages/with_links/data_stream/foo/manifest.yml

@@ -1,7 +1,7 @@
 title: Nginx access logs
 type: logs
 streams:
-  - input: logfile
+  - input: logfile-link
     vars:
       - name: empty_array
         type: text

test/packages/with_links/manifest.yml

@@ -14,6 +14,20 @@ policy_templates:
     title: Apache logs and metrics
     description: Collect logs and metrics from Apache instances
     inputs:
+      - type: logfile-link
+        title: Collect logs from Apache instances
+        description: Collecting Apache access and error logs
+        multi: false
+        vars:
+          - name: paths
+            type: text
+            title: Paths
+            multi: true
+            required: true
+            show_user: true
+            default:
+              - /var/log/apache2/access.log
+              - /var/log/apache2/error.log
       - type: apache/metrics
         title: Collect metrics from Apache instances
         description: Collecting Apache status metrics