render templates with external functions (#1876)

Some consumers of `consul-template` use it like a library, where the application
runs the runner in-process. For projects like Nomad which need to run with a
high level of privilege, this is problematic in that its challenging to secure
the operations that read and write from disk without running the entirety of CT
as an external process (which carries a lot of overhead for Nomad workloads).

Add a `RendererFunc` and `ReaderFunc` interface to allow Nomad to inject a
sandboxed subprocess when reading from disk and writing to disk.

This implementation is currently being used in Nomad 1.7.4, 1.6.7, and
1.5.14 as a mitigation for hashicorp/nomad#19888. See
hashicorp/nomad@df86503
for example usage.

cli_test.go

@@ -909,6 +909,16 @@ func TestCLI_ParseFlags(t *testing.T) {
 				e.Finalize()
 			}
 
+			// these can't be compared with DeepEqual
+			if e != nil {
+				e.RendererFunc = nil
+				e.ReaderFunc = nil
+			}
+			if a != nil {
+				a.RendererFunc = nil
+				a.ReaderFunc = nil
+			}
+
 			if !reflect.DeepEqual(e, a) {
 				t.Errorf("Config diff: %soutput: %q", e.Diff(a), out)
 			}

config/config.go

@@ -14,6 +14,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/hashicorp/consul-template/renderer"
 	"github.com/hashicorp/consul-template/signals"
 	"github.com/hashicorp/hcl"
 	homedir "github.com/mitchellh/go-homedir"
@@ -111,8 +112,24 @@ type Config struct {
 	// ErrOnFailedLookup, when enabled, will trigger an error if a dependency
 	// fails to return a value.
 	ErrOnFailedLookup bool `mapstructure:"err_on_failed_lookup"`
+
+	// RendererFunc is called whenever the template needs to be written, and
+	// will default to renderer.Render. This is intended for use when embedding
+	// Consul Template in another application
+	RendererFunc renderer.Renderer `mapstructure:"-" json:"-"`
+
+	// ReaderFunc is called whenever the template source is read, and will
+	// default to os.ReadFile. This is intended for use when embedding Consul
+	// Template in another application.
+	ReaderFunc Reader `mapstructure:"-" json:"-"`
 }
 
+// Reader is an interface that is implemented by os.OpenFile. The
+// Config.ReaderFunc requires this interface so that applications that embed
+// Consul Template can have an alternative implementation of os.OpenFile
+// (ex. virtual file, sandboxed reads)
+type Reader func(src string) ([]byte, error)
+
 // Copy returns a deep copy of the current configuration. This is useful because
 // the nested data structures may be shared.
 func (c *Config) Copy() *Config {
@@ -182,6 +199,9 @@ func (c *Config) Copy() *Config {
 		o.Nomad = c.Nomad.Copy()
 	}
 
+	o.RendererFunc = c.RendererFunc
+	o.ReaderFunc = c.ReaderFunc
+
 	return &o
 }
 
@@ -275,6 +295,13 @@ func (c *Config) Merge(o *Config) *Config {
 		r.Nomad = r.Nomad.Merge(o.Nomad)
 	}
 
+	if o.RendererFunc != nil {
+		r.RendererFunc = o.RendererFunc
+	}
+	if o.ReaderFunc != nil {
+		r.ReaderFunc = o.ReaderFunc
+	}
+
 	return r
 }
 
@@ -636,6 +663,13 @@ func (c *Config) Finalize() {
 	if c.BlockQueryWaitTime == nil {
 		c.BlockQueryWaitTime = TimeDuration(DefaultBlockQueryWaitTime)
 	}
+
+	if c.RendererFunc == nil {
+		c.RendererFunc = renderer.Render
+	}
+	if c.ReaderFunc == nil {
+		c.ReaderFunc = os.ReadFile
+	}
 }
 
 func stringFromEnv(list []string, def string) *string {

config/config_test.go

@@ -2616,6 +2616,12 @@ func TestDefaultConfig(t *testing.T) {
 			c := DefaultConfig()
 			c.Finalize()
 
+			// these can't be compared with DeepEqual
+			c.RendererFunc = nil
+			c.ReaderFunc = nil
+			r.RendererFunc = nil
+			r.ReaderFunc = nil
+
 			if !reflect.DeepEqual(r, c) {
 				t.Errorf("Config diff: %s", r.Diff(c))
 			}

manager/runner.go

@@ -114,6 +114,16 @@ type Runner struct {
 	// stopped is a boolean of whether the runner is stopped
 	stopped bool
 
+	// rendererFn is called whenever the template needs to be written, and will
+	// default to renderer.Render. This is intended for use when embedding
+	// Consul Template in another application
+	rendererFn renderer.Renderer
+
+	// readerFn is called whenever the template source is read, and will default
+	// to os.ReadFile. This is intended for use when embedding Consul Template
+	// in another application.
+	readerFn config.Reader
+
 	// finalConfigCopy provides access to a static copy of the finalized
 	// Runner config. This prevents risk of data races when reading config for
 	// other elements started by the Runner, like template functions.
@@ -200,6 +210,15 @@ func NewRunner(config *config.Config, dry bool) (*Runner, error) {
 		brain:         template.NewBrain(),
 		quiescenceMap: make(map[string]*quiescence),
 		quiescenceCh:  make(chan *template.Template),
+		rendererFn:    config.RendererFunc,
+		readerFn:      config.ReaderFunc,
+	}
+
+	if runner.rendererFn == nil {
+		runner.rendererFn = renderer.Render
+	}
+	if runner.readerFn == nil {
+		runner.readerFn = os.ReadFile
 	}
 
 	// Create the clientset
@@ -853,7 +872,7 @@ func (r *Runner) runTemplate(tmpl *template.Template, runCtx *templateRunCtx) (*
 		log.Printf("[DEBUG] (runner) rendering %s", templateConfig.Display())
 
 		// Render the template, taking dry mode into account
-		result, err := renderer.Render(&renderer.RenderInput{
+		result, err := r.rendererFn(&renderer.RenderInput{
 			Backup:         config.BoolVal(templateConfig.Backup),
 			Contents:       result.Output,
 			CreateDestDirs: config.BoolVal(templateConfig.CreateDestDirs),
@@ -975,6 +994,7 @@ func (r *Runner) init(clients *dep.ClientSet) error {
 			SandboxPath:      config.StringVal(ctmpl.SandboxPath),
 			Destination:      config.StringVal(ctmpl.Destination),
 			Config:           ctmpl,
+			ReaderFunc:       r.config.ReaderFunc,
 		})
 		if err != nil {
 			return err

renderer/renderer.go

@@ -59,6 +59,8 @@ type RenderResult struct {
 	Contents []byte
 }
 
+type Renderer func(*RenderInput) (*RenderResult, error)
+
 // Render atomically renders a file contents to disk, returning a result of
 // whether it would have rendered and actually did render.
 func Render(i *RenderInput) (*RenderResult, error) {

template/template.go

@@ -8,7 +8,6 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"os"
 	"strings"
 	"text/template"
 
@@ -28,6 +27,10 @@ var (
 	// does not specify either a "source" or "content" argument, which is not
 	// valid.
 	ErrTemplateMissingContentsAndSource = errors.New("template: must specify exactly one of 'source' or 'contents'")
+
+	// ErrMissingReaderFunction is the error returned when the template
+	// configuration is missing a reader function.
+	ErrMissingReaderFunction = errors.New("template: missing a reader function")
 )
 
 // Template is the internal representation of an individual template to process.
@@ -117,6 +120,9 @@ type NewTemplateInput struct {
 
 	// Config keeps local reference to config struct
 	Config *config.TemplateConfig
+
+	// ReaderFunc is called to read in any source file
+	ReaderFunc config.Reader
 }
 
 // NewTemplate creates and parses a new Consul Template template at the given
@@ -154,7 +160,10 @@ func NewTemplate(i *NewTemplateInput) (*Template, error) {
 	}
 
 	if i.Source != "" {
-		contents, err := os.ReadFile(i.Source)
+		if i.ReaderFunc == nil {
+			return nil, ErrMissingReaderFunction
+		}
+		contents, err := i.ReaderFunc(i.Source)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to read template")
 		}

template/template_test.go

@@ -119,6 +119,9 @@ func TestNewTemplate(t *testing.T) {
 
 	for i, tc := range cases {
 		t.Run(fmt.Sprintf("%d_%s", i, tc.name), func(t *testing.T) {
+			if tc.i != nil {
+				tc.i.ReaderFunc = os.ReadFile
+			}
 			a, err := NewTemplate(tc.i)
 			if (err != nil) != tc.err {
 				t.Fatal(err)