Skip to content

Commit

Permalink
Added templatestring function similar to templatefile (opentofu#1223)
Browse files Browse the repository at this point in the history
Signed-off-by: sanskruti-shahu <sanskruti.shahu@harness.io>
Signed-off-by: Sanskruti Shahu <76054960+sanskruti-shahu@users.noreply.github.com>
Co-authored-by: James Humphries <James@james-humphries.co.uk>
Signed-off-by: Ashwin Annamalai <4549937+IgnorantSapient@users.noreply.github.com>
  • Loading branch information
2 people authored and IgnorantSapient committed Apr 1, 2024
1 parent 8667910 commit e31b5e7
Show file tree
Hide file tree
Showing 12 changed files with 517 additions and 60 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ UPGRADE NOTES:
NEW FEATURES:

ENHANCEMENTS:
* Added `templatestring` function that takes a string and renders it as a template using a supplied set of template variables. ([#1223](https://github.com/opentofu/opentofu/pull/1223))
* Added `-concise` flag to omit the refreshing state logs when tofu plan is run. ([#1225](https://github.com/opentofu/opentofu/pull/1225))
* `nonsensitive` function no longer returns error when applied to values that are not sensitive ([#369](https://github.com/opentofu/opentofu/pull/369))
* Managing large local terraform.tfstate files is now much faster. ([#579](https://github.com/opentofu/opentofu/pull/579))
Expand Down
4 changes: 4 additions & 0 deletions internal/lang/funcs/descriptions.go
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,10 @@ var DescriptionList = map[string]descriptionEntry{
Description: "`templatefile` reads the file at the given path and renders its content as a template using a supplied set of template variables.",
ParamDescription: []string{"", ""},
},
"templatestring": {
Description: "`templatestring` processes the provided string as a template using a supplied set of template variables.",
ParamDescription: []string{"", ""},
},
"textdecodebase64": {
Description: "`textdecodebase64` function decodes a string that was previously Base64-encoded, and then interprets the result as characters in a specified character encoding.",
ParamDescription: []string{"", ""},
Expand Down
61 changes: 2 additions & 59 deletions internal/lang/funcs/filesystem.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,63 +101,6 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
return expr, nil
}

renderTmpl := func(expr hcl.Expression, varsVal cty.Value) (cty.Value, error) {
if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
}

ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
}

// We require all of the variables to be valid HCL identifiers, because
// otherwise there would be no way to refer to them in the template
// anyway. Rejecting this here gives better feedback to the user
// than a syntax error somewhere in the template itself.
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
// This error message intentionally doesn't describe _all_ of
// the different permutations that are technically valid as an
// HCL identifier, but rather focuses on what we might
// consider to be an "idiomatic" variable name.
return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
}
}

// We'll pre-check references in the template here so we can give a
// more specialized error message than HCL would by default, so it's
// clearer that this problem is coming from a templatefile call.
for _, traversal := range expr.Variables() {
root := traversal.RootName()
if _, ok := ctx.Variables[root]; !ok {
return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %q, referenced at %s", root, traversal[0].SourceRange())
}
}

givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
funcs := make(map[string]function.Function, len(givenFuncs))
for name, fn := range givenFuncs {
if name == "templatefile" {
// We stub this one out to prevent recursive calls.
funcs[name] = function.New(&function.Spec{
Params: params,
Type: func(args []cty.Value) (cty.Type, error) {
return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile call")
},
})
continue
}
funcs[name] = fn
}
ctx.Functions = funcs

val, diags := expr.Value(ctx)
if diags.HasErrors() {
return cty.DynamicVal, diags
}
return val, nil
}

return function.New(&function.Spec{
Params: params,
Type: func(args []cty.Value) (cty.Type, error) {
Expand All @@ -177,7 +120,7 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun

// This is safe even if args[1] contains unknowns because the HCL
// template renderer itself knows how to short-circuit those.
val, err := renderTmpl(expr, args[1])
val, err := renderTemplate(expr, args[1], funcsCb)
return val.Type(), err
},
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
Expand All @@ -186,7 +129,7 @@ func MakeTemplateFileFunc(baseDir string, funcsCb func() map[string]function.Fun
if err != nil {
return cty.DynamicVal, err
}
result, err := renderTmpl(expr, args[1])
result, err := renderTemplate(expr, args[1], funcsCb)
return result.WithMarks(pathMarks), err
},
})
Expand Down
2 changes: 1 addition & 1 deletion internal/lang/funcs/filesystem_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,7 @@ func TestTemplateFile(t *testing.T) {
cty.StringVal("testdata/recursive.tmpl"),
cty.MapValEmpty(cty.String),
cty.NilVal,
`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile call.`,
`testdata/recursive.tmpl:1,3-16: Error in function call; Call to function "templatefile" failed: cannot recursively call templatefile from inside templatefile or templatestring.`,
},
{
cty.StringVal("testdata/list.tmpl"),
Expand Down
89 changes: 89 additions & 0 deletions internal/lang/funcs/render_template.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package funcs

import (
"fmt"

"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)

func renderTemplate(expr hcl.Expression, varsVal cty.Value, funcsCb func() map[string]function.Function) (cty.Value, error) {
if varsTy := varsVal.Type(); !(varsTy.IsMapType() || varsTy.IsObjectType()) {
return cty.DynamicVal, function.NewArgErrorf(1, "invalid vars value: must be a map") // or an object, but we don't strongly distinguish these most of the time
}

ctx := &hcl.EvalContext{
Variables: varsVal.AsValueMap(),
}

// We require all of the variables to be valid HCL identifiers, because
// otherwise there would be no way to refer to them in the template
// anyway. Rejecting this here gives better feedback to the user
// than a syntax error somewhere in the template itself.
for n := range ctx.Variables {
if !hclsyntax.ValidIdentifier(n) {
// This error message intentionally doesn't describe _all_ of
// the different permutations that are technically valid as an
// HCL identifier, but rather focuses on what we might
// consider to be an "idiomatic" variable name.
return cty.DynamicVal, function.NewArgErrorf(1, "invalid template variable name %q: must start with a letter, followed by zero or more letters, digits, and underscores", n)
}
}

// currFilename stores the filename of the template file, if any.
currFilename := expr.Range().Filename

// We'll pre-check references in the template here so we can give a
// more specialized error message than HCL would by default, so it's
// clearer that this problem is coming from a templatefile/templatestring call.
for _, traversal := range expr.Variables() {
root := traversal.RootName()
referencedPos := fmt.Sprintf("%q", root)
if currFilename != templateStringFilename {
referencedPos = fmt.Sprintf("%q, referenced at %s", root, traversal[0].SourceRange())
}
if _, ok := ctx.Variables[root]; !ok {
return cty.DynamicVal, function.NewArgErrorf(1, "vars map does not contain key %s", referencedPos)
}
}

givenFuncs := funcsCb() // this callback indirection is to avoid chicken/egg problems
funcs := make(map[string]function.Function, len(givenFuncs))
for name, fn := range givenFuncs {
if name == "templatefile" {
// We stub this one out to prevent recursive calls.
funcs[name] = function.New(&function.Spec{
Params: []function.Parameter{
{
Name: "path",
Type: cty.String,
AllowMarked: true,
},
{
Name: "vars",
Type: cty.DynamicPseudoType,
},
},
Type: func(args []cty.Value) (cty.Type, error) {
return cty.NilType, fmt.Errorf("cannot recursively call templatefile from inside templatefile or templatestring")
},
})
continue
}
funcs[name] = fn
}
ctx.Functions = funcs

val, diags := expr.Value(ctx)
if diags.HasErrors() {
return cty.DynamicVal, diags
}
return val, nil
}
112 changes: 112 additions & 0 deletions internal/lang/funcs/render_template_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,112 @@
// Copyright (c) The OpenTofu Authors
// SPDX-License-Identifier: MPL-2.0
// Copyright (c) 2023 HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0

package funcs

import (
"testing"

"github.com/hashicorp/hcl/v2"
"github.com/opentofu/opentofu/internal/lang/marks"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)

func TestRenderTemplate(t *testing.T) {
tests := map[string]struct {
Expr hcl.Expression
Vars cty.Value
Want cty.Value
Err string
}{
"String interpolation with variable": {
hcl.StaticExpr(cty.StringVal("Hello, ${name}!"), hcl.Range{}),
cty.MapVal(map[string]cty.Value{
"name": cty.StringVal("Jodie"),
}),
cty.StringVal("Hello, ${name}!"),
``,
},
"Looping through list": {
hcl.StaticExpr(cty.StringVal("Items: %{ for x in list ~} ${x} %{ endfor ~}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"list": cty.ListVal([]cty.Value{
cty.StringVal("a"),
cty.StringVal("b"),
cty.StringVal("c"),
}),
}),
cty.StringVal("Items: %{ for x in list ~} ${x} %{ endfor ~}"),
``,
},
"Looping through map": {
hcl.StaticExpr(cty.StringVal("%{ for key, value in list ~} ${key}:${value} %{ endfor ~}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"list": cty.ObjectVal(map[string]cty.Value{
"item1": cty.StringVal("a"),
"item2": cty.StringVal("b"),
"item3": cty.StringVal("c"),
}),
}),
cty.StringVal("%{ for key, value in list ~} ${key}:${value} %{ endfor ~}"),
``,
},
"Invalid template variable name": {
hcl.StaticExpr(cty.StringVal("Hello, ${1}!"), hcl.Range{}),
cty.MapVal(map[string]cty.Value{
"1": cty.StringVal("Jodie"),
}),
cty.NilVal,
`invalid template variable name "1": must start with a letter, followed by zero or more letters, digits, and underscores`,
},
"Interpolation of a boolean value": {
hcl.StaticExpr(cty.StringVal("${val}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"val": cty.True,
}),
cty.StringVal("${val}"),
``,
},
"Sensitive string template": {
hcl.StaticExpr(cty.StringVal("My password is 1234").Mark(marks.Sensitive), hcl.Range{}),
cty.EmptyObjectVal,
cty.StringVal("My password is 1234").Mark(marks.Sensitive),
``,
},
"Sensitive template variable": {
hcl.StaticExpr(cty.StringVal("My password is ${pass}"), hcl.Range{}),
cty.ObjectVal(map[string]cty.Value{
"pass": cty.StringVal("secret").Mark(marks.Sensitive),
}),
cty.StringVal("My password is ${pass}"),
``,
},
}

for name, test := range tests {
t.Run(name, func(t *testing.T) {

got, err := renderTemplate(test.Expr, test.Vars, func() map[string]function.Function {
return map[string]function.Function{}
})

if err != nil {
if test.Err == "" {
t.Fatalf("unexpected error: %s", err)
} else {
if got, want := err.Error(), test.Err; got != want {
t.Errorf("wrong error\ngot: %s\nwant: %s", got, want)
}
}
} else if test.Err != "" {
t.Fatal("succeeded; want error")
} else {
if !got.RawEquals(test.Want) {
t.Errorf("wrong result\ngot: %#v\nwant: %#v", got, test.Want)
}
}
})
}
}
68 changes: 68 additions & 0 deletions internal/lang/funcs/string.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (
"regexp"
"strings"

"github.com/hashicorp/hcl/v2"
"github.com/hashicorp/hcl/v2/hclsyntax"
"github.com/zclconf/go-cty/cty"
"github.com/zclconf/go-cty/cty/function"
)
Expand Down Expand Up @@ -164,3 +166,69 @@ func Replace(str, substr, replace cty.Value) (cty.Value, error) {
func StrContains(str, substr cty.Value) (cty.Value, error) {
return StrContainsFunc.Call([]cty.Value{str, substr})
}

// This constant provides a placeholder value for filename indicating
// that no file is needed for templatestring.
const (
templateStringFilename = "NoFileNeeded"
)

// MakeTemplateStringFunc constructs a function that takes a string and
// an arbitrary object of named values and attempts to render that string
// as a template using HCL template syntax.
func MakeTemplateStringFunc(content string, funcsCb func() map[string]function.Function) function.Function {

params := []function.Parameter{
{
Name: "data",
Type: cty.String,
AllowMarked: true,
},
{
Name: "vars",
Type: cty.DynamicPseudoType,
AllowMarked: true,
},
}
loadTmpl := func(content string, marks cty.ValueMarks) (hcl.Expression, error) {

expr, diags := hclsyntax.ParseTemplate([]byte(content), templateStringFilename, hcl.Pos{Line: 1, Column: 1})
if diags.HasErrors() {
return nil, diags
}

return expr, nil
}

return function.New(&function.Spec{
Params: params,
Type: func(args []cty.Value) (cty.Type, error) {
if !(args[0].IsKnown() && args[1].IsKnown()) {
return cty.DynamicPseudoType, nil
}

// We'll render our template now to see what result type it produces.
// A template consisting only of a single interpolation can potentially
// return any type.
dataArg, dataMarks := args[0].Unmark()
expr, err := loadTmpl(dataArg.AsString(), dataMarks)
if err != nil {
return cty.DynamicPseudoType, err
}

// This is safe even if args[1] contains unknowns because the HCL
// template renderer itself knows how to short-circuit those.
val, err := renderTemplate(expr, args[1], funcsCb)
return val.Type(), err
},
Impl: func(args []cty.Value, retType cty.Type) (cty.Value, error) {
dataArg, dataMarks := args[0].Unmark()
expr, err := loadTmpl(dataArg.AsString(), dataMarks)
if err != nil {
return cty.DynamicVal, err
}
result, err := renderTemplate(expr, args[1], funcsCb)
return result.WithMarks(dataMarks), err
},
})
}

0 comments on commit e31b5e7

Please sign in to comment.