HCL2: add templatefile function (#10776)

* tests
* docs

.gitattributes

@@ -6,6 +6,7 @@
 *.mdx text eol=lf
 *.ps1 text eol=lf
 *.hcl text eol=lf
+*.tmpl text eol=lf
 *.txt text eol=lf
 go.mod text eol=lf
 go.sum text eol=lf

hcl2template/function/templatefile.go

@@ -0,0 +1,140 @@
+package function
+
+import (
+	"fmt"
+
+	"github.com/hashicorp/go-cty-funcs/filesystem"
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+)
+
+// MakeTemplateFileFunc constructs a function that takes a file path and
+// an arbitrary object of named values and attempts to render the referenced
+// file as a template using HCL template syntax.
+//
+// The template itself may recursively call other functions so a callback
+// must be provided to get access to those functions. The template cannot,
+// however, access any variables defined in the scope: it is restricted only to
+// those variables provided in the second function argument.
+//
+// As a special exception, a referenced template file may not recursively call
+// the templatefile function, since that would risk the same file being
+// included into itself indefinitely.
+func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Function) function.Function {
+
+	params := []function.Parameter{
+		{
+			Name: "path",
+			Type: cty.String,
+		},
+		{
+			Name: "vars",
+			Type: cty.DynamicPseudoType,
+		},
+	}
+
+	loadTmpl := func(fn string) (hcl.Expression, error) {
+		// We re-use File here to ensure the same filename interpretation
+		// as it does, along with its other safety checks.
+		tmplVal, err := filesystem.File(baseDir, cty.StringVal(fn))
+		if err != nil {
+			return nil, err
+		}
+
+		expr, diags := hclsyntax.ParseTemplate([]byte(tmplVal.AsString()), fn, hcl.Pos{Line: 1, Column: 1})
+		if diags.HasErrors() {
+			return nil, diags
+		}
+
+		return expr, nil
+	}
+
+	renderTmpl := func(expr hcl.Expression, varsVal cty.Value) (cty.Value, error) {
+		if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
+			return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
+		}
+
+		ctx := &hcl.EvalContext{
+			Variables: varsVal.AsValueMap(),
+		}
+
+		// We require all of the variables to be valid HCL identifiers, because
+		// otherwise there would be no way to refer to them in the template
+		// anyway. Rejecting this here gives better feedback to the user
+		// than a syntax error somewhere in the template itself.
+		for n := range ctx.Variables {
+			if !hclsyntax.ValidIdentifier(n) {
+				// This error message intentionally doesn't describe _all_ of
+				// the different permutations that are technically valid as an
+				// HCL identifier, but rather focuses on what we might
+				// consider to be an "idiomatic" variable name.
+				return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
+			}
+		}
+
+		// We'll pre-check references in the template here so we can give a
+		// more specialized error message than HCL would by default, so it's
+		// clearer that this problem is coming from a templatefile call.
+		for _, traversal := range expr.Variables() {
+			root := traversal.RootName()
+			if _, ok := ctx.Variables[root]; !ok {
+				return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
+			}
+		}
+
+		givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
+		funcs := make(map[string]function.Function, len(givenFuncs))
+		for name, fn := range givenFuncs {
+			if name == "templatefile" {
+				// We stub this one out to prevent recursive calls.
+				funcs[name] = function.New(&function.Spec{
+					Params: params,
+					Type: func(args []cty.Value) (cty.Type, error) {
+						return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile call")
+					},
+				})
+				continue
+			}
+			funcs[name] = fn
+		}
+		ctx.Functions = funcs
+
+		val, diags := expr.Value(ctx)
+		if diags.HasErrors() {
+			return cty.DynamicVal, diags
+		}
+		return val, nil
+	}
+
+	return function.New(&function.Spec{
+		Params: params,
+		Type: func(args []cty.Value) (cty.Type, error) {
+			if !(args[0].IsKnown() && args[1].IsKnown()) {
+				return cty.DynamicPseudoType, nil
+			}
+
+			// We'll render our template now to see what result type it
+			// produces. A template consisting only of a single interpolation
+			// can potentially return any type.
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicPseudoType, err
+			}
+
+			// This is safe even if args[1] contains unknowns because the HCL
+			// template renderer itself knows how to short-circuit those.
+			val, err := renderTmpl(expr, args[1])
+			return val.Type(), err
+		},
+		Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
+			expr, err := loadTmpl(args[0].AsString())
+			if err != nil {
+				return cty.DynamicVal, err
+			}
+			return renderTmpl(expr, args[1])
+		},
+	})
+
+}

hcl2template/function/templatefile_test.go

@@ -0,0 +1,151 @@
+package function
+
+import (
+	"fmt"
+	"path/filepath"
+	"testing"
+
+	"github.com/hashicorp/go-cty-funcs/filesystem"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/function"
+	"github.com/zclconf/go-cty/cty/function/stdlib"
+)
+
+func TestTemplateFile(t *testing.T) {
+	tests := []struct {
+		Path cty.Value
+		Vars cty.Value
+		Want cty.Value
+		Err  string
+	}{
+		{
+			cty.StringVal("testdata/hello.txt"),
+			cty.EmptyObjectVal,
+			cty.StringVal("Hello World"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/icon.png"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`contents of testdata/icon.png are not valid UTF-8; use the filebase64 function to obtain the Base64 encoded contents or the other file functions (e.g. filemd5, filesha256) to obtain file hashing results instead`,
+		},
+		{
+			cty.StringVal("testdata/missing"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`no file exists at ` + filepath.Clean("testdata/missing"),
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.MapVal(map[string]cty.Value{
+				"name": cty.StringVal("Jodie"),
+			}),
+			cty.StringVal("Hello, Jodie!"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.MapVal(map[string]cty.Value{
+				"name!": cty.StringVal("Jodie"),
+			}),
+			cty.NilVal,
+			`invalid template variable name "name!": must start with a letter, followed by zero or more letters, digits, and underscores`,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"name": cty.StringVal("Jimbo"),
+			}),
+			cty.StringVal("Hello, Jimbo!"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/hello.tmpl"),
+			cty.EmptyObjectVal,
+			cty.NilVal,
+			`vars map does not contain key "name", referenced at testdata/hello.tmpl:1,10-14`,
+		},
+		{
+			cty.StringVal("testdata/func.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("The items are a, b, c"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/recursive.tmpl"),
+			cty.MapValEmpty(cty.String),
+			cty.NilVal,
+			`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile call.`,
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.ListVal([]cty.Value{
+					cty.StringVal("a"),
+					cty.StringVal("b"),
+					cty.StringVal("c"),
+				}),
+			}),
+			cty.StringVal("- a\n- b\n- c\n"),
+			``,
+		},
+		{
+			cty.StringVal("testdata/list.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"list": cty.True,
+			}),
+			cty.NilVal,
+			`testdata/list.tmpl:1,13-17: Iteration over non-iterable value; A value of type bool cannot be used as the collection in a 'for' expression.`,
+		},
+		{
+			cty.StringVal("testdata/bare.tmpl"),
+			cty.ObjectVal(map[string]cty.Value{
+				"val": cty.True,
+			}),
+			cty.True, // since this template contains only an interpolation, its true value shines through
+			``,
+		},
+	}
+
+	templateFileFn := MakeTemplateFileFunc(".", func() map[string]function.Function {
+		return map[string]function.Function{
+			"join":         stdlib.JoinFunc,
+			"templatefile": filesystem.MakeFileFunc(".", false), // just a placeholder, since templatefile itself overrides this
+		}
+	})
+
+	for _, test := range tests {
+		t.Run(fmt.Sprintf("TemplateFile(%#v, %#v)", test.Path, test.Vars), func(t *testing.T) {
+			got, err := templateFileFn.Call([]cty.Value{test.Path, test.Vars})
+
+			if argErr, ok := err.(function.ArgError); ok {
+				if argErr.Index < 0 || argErr.Index > 1 {
+					t.Errorf("ArgError index %d is out of range for templatefile (must be 0 or 1)", argErr.Index)
+				}
+			}
+
+			if test.Err != "" {
+				if err == nil {
+					t.Fatal("succeeded; want error")
+				}
+				if got, want := err.Error(), test.Err; got != want {
+					t.Errorf("wrong error\ngot:  %s\nwant: %s", got, want)
+				}
+				return
+			} else if err != nil {
+				t.Fatalf("unexpected error: %s", err)
+			}
+
+			if !got.RawEquals(test.Want) {
+				t.Errorf("wrong result\ngot:  %#v\nwant: %#v", got, test.Want)
+			}
+		})
+	}
+}

hcl2template/function/testdata/bare.tmpl

@@ -0,0 +1 @@
+${val}

hcl2template/function/testdata/func.tmpl

@@ -0,0 +1 @@
+The items are ${join(", ", list)}

hcl2template/function/testdata/hello.tmpl

@@ -0,0 +1 @@
+Hello, ${name}!

hcl2template/function/testdata/hello.txt

@@ -0,0 +1 @@
+Hello World

hcl2template/function/testdata/list.tmpl

@@ -0,0 +1,3 @@
+%{ for x in list ~}
+- ${x}
+%{ endfor ~}

hcl2template/function/testdata/recursive.tmpl

@@ -0,0 +1 @@
+${templatefile("recursive.tmpl", {})}

hcl2template/functions.go

@@ -118,6 +118,12 @@ func Functions(basedir string) map[string]function.Function {
 		"zipmap":             stdlib.ZipmapFunc,
 	}
 
+	funcs["templatefile"] = pkrfunction.MakeTemplateFileFunc(basedir, func() map[string]function.Function {
+		// The templatefile function prevents recursive calls to itself
+		// by copying this map and overwriting the "templatefile" entry.
+		return funcs
+	})
+
 	return funcs
 }
 

website/content/docs/templates/hcl_templates/functions/file/fileexists.mdx

@@ -32,4 +32,5 @@ fileexists("custom-section.sh") ? file("custom-section.sh") : local.default_cont
 
 ## Related Functions
 
-- [`file`](/docs/templates/hcl_templates/functions/file) reads the contents of a file at a given path
+- [`file`](/docs/templates/hcl_templates/functions/file/file) reads the contents
+  of a file at a given path.