fix: Reworking dirReplacements system to work with SetPath and Blocks

internal/codegen/stencil.go

@@ -5,13 +5,15 @@
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"math/rand"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
 	"sort"
+	"strings"
 	"time"
 
 	"github.com/go-git/go-billy/v5/util"
@@ -225,6 +227,10 @@
 	// Sort module hook data before the next pass
 	s.sortModuleHooks()
 
+	if err := s.calcDirReplacements(vals); err != nil {
+		return nil, err
+	}
+
 	tpls := make([]*Template, 0)
 	for _, t := range tplfiles {
 		log.Debugf("Second pass render of template %s", t.ImportPath())
@@ -239,6 +245,44 @@
 	return tpls, nil
 }
 
+// calcDirReplacements calculates all of the final rendered paths for dirReplacements for each module
+// It needs to be in stencil because it uses rendering, which needs the Values object from codegen,
+// so we poke the rendered replacements into the module object for applying later in various ways.
+func (s *Stencil) calcDirReplacements(vals *Values) error {
+	for _, m := range s.modules {
+		reps := map[string]string{}
+		for dsrc, dtmp := range m.Manifest.DirReplacements {
+			// Render replacement
+			nn, err := s.renderDirReplacement(dtmp, m, vals)
+			if err != nil {
+				return err
+			}
+			reps[dsrc] = nn
+		}
+		m.StoreDirReplacements(reps)
+	}
+	return nil
+}
+
+// renderDirReplacement breaks out the actual rendering for calcDirReplacements to make it unit testable
+func (s *Stencil) renderDirReplacement(template string, m *modules.Module, vals *Values) (string, error) {
+	rt, err := NewTemplate(m, "dirReplace", 0o000, time.Time{}, []byte(template), s.log)
+	if err != nil {
+		return "", err
+	}
+
+	if err := rt.Render(s, vals); err != nil {
+		return "", err
+	}
+
+	nn := rt.Files[0].String()
+	if strings.Contains(nn, string(os.PathSeparator)) {
+		return "", fmt.Errorf("directory replacement of %s to %s contains path separator in output", template, nn)
+	}
+
+	return nn, nil
+}
+
 // PostRun runs all post run commands specified in the modules that
 // this project depends on
 func (s *Stencil) PostRun(ctx context.Context, log slogext.Logger) error {

internal/codegen/stencil_test.go

@@ -106,3 +106,39 @@ func TestModuleHookRender(t *testing.T) {
 	assert.Equal(t, len(tpls[1].Files), 1, "expected Render() m2 template to return a single file")
 	assert.Equal(t, strings.TrimSpace(tpls[1].Files[0].String()), "a", "expected Render() m2 to return correct output")
 }
+
+func TestDirReplacementRendering(t *testing.T) {
+	log := slogext.NewTestLogger(t)
+	sm := &configuration.Manifest{Name: "testing", Arguments: map[string]any{"x": "d"}}
+	m1man := &configuration.TemplateRepositoryManifest{
+		Name: "testing1",
+		DirReplacements: map[string]string{
+			"testdata":             `bob`,
+			"testdata/replacement": `{{ stencil.Arg "x" }}`,
+		},
+		Arguments: map[string]configuration.Argument{"x": {Schema: map[string]any{"type": "string"}}},
+	}
+	m1, err := modulestest.NewModuleFromTemplates(m1man, "testdata/replacement/m1.tpl")
+	assert.NilError(t, err, "failed to NewWithFS")
+
+	st := NewStencil(sm, []*modules.Module{m1}, log)
+
+	tps, err := st.Render(context.Background(), log)
+	assert.NilError(t, err, "failed to render template")
+	assert.Equal(t, len(tps), 1)
+	assert.Equal(t, len(tps[0].Files), 1)
+	assert.Equal(t, tps[0].Files[0].path, "bob/d/m1")
+}
+
+func TestBadDirReplacement(t *testing.T) {
+	log := slogext.NewTestLogger(t)
+	sm := &configuration.Manifest{Name: "testing"}
+	m1man := &configuration.TemplateRepositoryManifest{Name: "testing1"}
+	m, err := modulestest.NewModuleFromTemplates(m1man, "testdata/replacement/m1.tpl")
+	assert.NilError(t, err, "failed to NewModuleFromTemplates")
+
+	st := NewStencil(sm, []*modules.Module{m}, log)
+	vals := NewValues(context.Background(), sm, nil)
+	_, err = st.renderDirReplacement("b/c", m, vals)
+	assert.ErrorContains(t, err, "contains path separator in output")
+}

internal/codegen/template.go

@@ -6,7 +6,6 @@ package codegen
 
 import (
 	"bytes"
-	"fmt"
 	"os"
 	"path"
 	"path/filepath"
@@ -100,7 +99,9 @@ func (t *Template) Parse(_ *Stencil) error {
 // are rendered onto the Files field of the template struct.
 func (t *Template) Render(st *Stencil, vals *Values) error {
 	if len(t.Files) == 0 && !t.Library {
-		f, err := NewFile(strings.TrimSuffix(t.Path, ".tpl"), t.mode, t.modTime)
+		p := strings.TrimSuffix(t.Path, ".tpl")
+		p = t.Module.ApplyDirReplacements(p)
+		f, err := NewFile(p, t.mode, t.modTime)
 		if err != nil {
 			return err
 		}
@@ -146,43 +147,5 @@ func (t *Template) Render(st *Stencil, vals *Values) error {
 		t.Files = t.Files[1:len(t.Files)]
 	}
 
-	if !st.isFirstPass {
-		// Now that everything's been decided, see if we need to replace any file paths from directory manifests
-		for _, tf := range t.Files {
-			if err := t.applyDirReplacements(tf, st, vals); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (t *Template) applyDirReplacements(tf *File, st *Stencil, vals *Values) error {
-	// Hop through the path dir by dir, starting at the end (because the raw paths won't match if you replace the earlier
-	// path segments first), and see if there's any replacements.
-	pp := strings.Split(tf.path, string(os.PathSeparator))
-	for i := len(pp) - 1; i >= 0; i-- {
-		pathPart := strings.Join(pp[0:i+1], string(os.PathSeparator))
-		if dr, has := t.Module.Manifest.DirReplacements[pathPart]; has {
-			// Render replacement
-			rt, err := NewTemplate(t.Module, "dirReplace", 0o000, time.Time{}, []byte(dr), t.log)
-			if err != nil {
-				return err
-			}
-
-			if err := rt.Render(st, vals); err != nil {
-				return err
-			}
-
-			nn := rt.Files[0].String()
-			if strings.Contains(nn, string(os.PathSeparator)) {
-				return fmt.Errorf("directory replacement of %s to %s contains path separator in output", pp[i], nn)
-			}
-			pp[i] = nn
-		}
-	}
-	tf.path = strings.Join(pp, string(os.PathSeparator))
-
 	return nil
 }

internal/codegen/template_test.go

@@ -13,11 +13,10 @@ import (
 
 	_ "embed"
 
-	"github.com/go-git/go-billy/v5"
-	"github.com/go-git/go-billy/v5/memfs"
 	"go.rgst.io/stencil/internal/modules"
 	"go.rgst.io/stencil/internal/modules/modulestest"
 	"go.rgst.io/stencil/internal/slogext"
+	"go.rgst.io/stencil/internal/testing/testmemfs"
 	"go.rgst.io/stencil/pkg/configuration"
 	"gotest.tools/v3/assert"
 )
@@ -37,17 +36,10 @@ var generatedBlockTemplate string
 //go:embed testdata/generated-block/fake.txt
 var fakeGeneratedBlockFile string
 
-func memfsWithManifest(manifest string) billy.Filesystem {
-	fs := memfs.New()
-	f, _ := fs.Create("manifest.yaml")
-	f.Write([]byte(manifest))
-	f.Close()
-	return fs
-}
-
 func TestSingleFileRender(t *testing.T) {
 	log := slogext.NewTestLogger(t)
-	fs := memfsWithManifest("name: testing\n")
+	fs, err := testmemfs.WithManifest("name: testing\n")
+	assert.NilError(t, err, "failed to testmemfs.WithManifest")
 	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
 	assert.NilError(t, err, "failed to NewWithFS")
 
@@ -64,7 +56,8 @@ func TestSingleFileRender(t *testing.T) {
 
 func TestMultiFileRender(t *testing.T) {
 	log := slogext.NewTestLogger(t)
-	fs := memfsWithManifest("name: testing\narguments:\n  commands:\n    type: list")
+	fs, err := testmemfs.WithManifest("name: testing\narguments:\n  commands:\n    type: list")
+	assert.NilError(t, err, "failed to testmemfs.WithManifest")
 	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
 	assert.NilError(t, err, "failed to NewWithFS")
 
@@ -88,7 +81,8 @@ func TestMultiFileRender(t *testing.T) {
 
 func TestMultiFileWithInputRender(t *testing.T) {
 	log := slogext.NewTestLogger(t)
-	fs := memfsWithManifest("name: testing\narguments:\n  commands:\n    type: list")
+	fs, err := testmemfs.WithManifest("name: testing\narguments:\n  commands:\n    type: list")
+	assert.NilError(t, err, "failed to testmemfs.WithManifest")
 	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
 	assert.NilError(t, err, "failed to NewWithFS")
 
@@ -112,7 +106,8 @@ func TestMultiFileWithInputRender(t *testing.T) {
 
 func TestApplyTemplateArgumentPassthrough(t *testing.T) {
 	log := slogext.NewTestLogger(t)
-	fs := memfsWithManifest("name: testing\narguments:\n  commands:\n    type: list")
+	fs, err := testmemfs.WithManifest("name: testing\narguments:\n  commands:\n    type: list")
+	assert.NilError(t, err, "failed to testmemfs.WithManifest")
 	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
 	assert.NilError(t, err, "failed to NewWithFS")
 
@@ -137,7 +132,8 @@ func TestGeneratedBlock(t *testing.T) {
 	tempDir := t.TempDir()
 	fakeFilePath := filepath.Join(tempDir, "generated-block.txt")
 
-	fs := memfsWithManifest("name: testing\n")
+	fs, err := testmemfs.WithManifest("name: testing\n")
+	assert.NilError(t, err, "failed to testmemfs.WithManifest")
 	sm := &configuration.Manifest{Name: "testing", Arguments: map[string]interface{}{}}
 	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
 	assert.NilError(t, err, "failed to NewWithFS")
@@ -163,7 +159,8 @@ func TestGeneratedBlock(t *testing.T) {
 // TestLibraryTemplate ensures that library templates don't generate
 // files as well as that the library flag is set correctly.
 func TestLibraryTemplate(t *testing.T) {
-	fs := memfsWithManifest("name: testing\n")
+	fs, err := testmemfs.WithManifest("name: testing\n")
+	assert.NilError(t, err, "failed to testmemfs.WithManifest")
 	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
 	log := slogext.NewTestLogger(t)
 	assert.NilError(t, err, "failed to NewWithFS")
@@ -183,7 +180,8 @@ func TestLibraryTemplate(t *testing.T) {
 // TestLibraryCantAccessFileFunctions ensures that library templates
 // can't access file functions in the template.
 func TestLibraryCantAccessFileFunctions(t *testing.T) {
-	fs := memfsWithManifest("name: testing\n")
+	fs, err := testmemfs.WithManifest("name: testing\n")
+	assert.NilError(t, err, "failed to testmemfs.WithManifest")
 	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
 	log := slogext.NewTestLogger(t)
 	assert.NilError(t, err, "failed to NewWithFS")
@@ -199,68 +197,3 @@ func TestLibraryCantAccessFileFunctions(t *testing.T) {
 		"expected library template to fail on render",
 	)
 }
-
-func TestSimpleDirReplacement(t *testing.T) {
-	log := slogext.NewTestLogger(t)
-	fs := memfsWithManifest("name: testing\ndirReplacements:\n  a: 'b'\n")
-	sm := &configuration.Manifest{Name: "testing", Arguments: map[string]interface{}{}}
-	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
-	assert.NilError(t, err, "failed to NewWithFS")
-
-	st := NewStencil(sm, []*modules.Module{m}, log)
-	tpl, err := NewTemplate(m, "a/base.tpl", 0o644, time.Now(), []byte("out"), log)
-	assert.NilError(t, err, "failed to create template")
-
-	tplf, err := NewFile("a/base", 0o644, time.Now())
-	assert.NilError(t, err, "failed to create file")
-
-	assert.Equal(t, tplf.path, "a/base")
-	vals := NewValues(context.Background(), sm, nil)
-	err = tpl.applyDirReplacements(tplf, st, vals)
-	assert.NilError(t, err, "failed to apply dir replacements")
-	assert.Equal(t, tplf.path, "b/base")
-}
-
-func TestNestedAndTemplatedDirReplacements(t *testing.T) {
-	log := slogext.NewTestLogger(t)
-	fs := memfsWithManifest(
-		"name: testing\ndirReplacements:\n  a: 'b'\n  a/c: '{{ stencil.Arg \"x\" }}'\narguments:\n  x:\n    type: string\n")
-	sm := &configuration.Manifest{Name: "testing", Arguments: map[string]any{"x": "d"}}
-	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
-	assert.NilError(t, err, "failed to NewWithFS")
-
-	st := NewStencil(sm, []*modules.Module{m}, log)
-	tpl, err := NewTemplate(m, "a/c/base.tpl", 0o644, time.Now(), []byte("out"), log)
-	assert.NilError(t, err, "failed to create template")
-
-	tplf, err := NewFile("a/c/base", 0o644, time.Now())
-	assert.NilError(t, err, "failed to create file")
-
-	assert.Equal(t, tplf.path, "a/c/base")
-	vals := NewValues(context.Background(), sm, nil)
-	err = tpl.applyDirReplacements(tplf, st, vals)
-	assert.NilError(t, err, "failed to apply dir replacements")
-	assert.Equal(t, tplf.path, "b/d/base")
-}
-
-func TestBadDirReplacement(t *testing.T) {
-	log := slogext.NewTestLogger(t)
-	fs := memfsWithManifest(
-		"name: testing\ndirReplacements:\n  a: 'b/c'\n")
-	sm := &configuration.Manifest{Name: "testing"}
-	m, err := modulestest.NewWithFS(context.Background(), "testing", fs)
-	assert.NilError(t, err, "failed to NewWithFS")
-
-	st := NewStencil(sm, []*modules.Module{m}, log)
-	tpl, err := NewTemplate(m, "a/base.tpl", 0o644, time.Now(), []byte("out"), log)
-	assert.NilError(t, err, "failed to create template")
-
-	tplf, err := NewFile("a/base", 0o644, time.Now())
-	assert.NilError(t, err, "failed to create file")
-
-	assert.Equal(t, tplf.path, "a/base")
-	vals := NewValues(context.Background(), sm, nil)
-	err = tpl.applyDirReplacements(tplf, st, vals)
-	assert.ErrorContains(t, err, "contains path separator in output")
-	assert.Equal(t, tplf.path, "a/base")
-}

internal/codegen/testdata/replacement/m1.tpl

@@ -0,0 +1 @@
+bob

internal/codegen/tpl_file.go

@@ -53,6 +53,7 @@
 //
 //	{{- file.SetPath "new/path/to/file.txt" }}
 func (f *TplFile) SetPath(path string) (out string, err error) {
+	path = f.t.Module.ApplyDirReplacements(path)
 	err = f.f.SetPath(path)
 	return "", err
 }

internal/modules/module.go

@@ -55,6 +55,11 @@ type Module struct {
 
 	// fs is a cached filesystem
 	fs billy.Filesystem
+
+	// dirReplacementsRendered is a rendered list of dirReplacements from the manifest,
+	// ready to be used for immediate replacements.  It's a mapping of relative paths
+	// to just the replacement name for the last path segment.
+	dirReplacementsRendered map[string]string
 }
 
 // uriIsLocal returns true if the URI is a local file path
@@ -212,3 +217,23 @@ func (m *Module) GetFS(ctx context.Context) (billy.Filesystem, error) {
 
 	return m.fs, nil
 }
+
+// StoreDirReplacements pokes the template-rendered output from the stencil render
+// function for use by the module rendering later on via ApplyDirReplacements.
+func (m *Module) StoreDirReplacements(reps map[string]string) {
+	m.dirReplacementsRendered = reps
+}
+
+// ApplyDirReplacements hops through the incoming path dir by dir, starting at the end
+// (because the raw paths won't match if you replace the earlier path segments first),
+// and see if there's any replacements to apply
+func (m *Module) ApplyDirReplacements(path string) string {
+	pp := strings.Split(path, string(os.PathSeparator))
+	for i := len(pp) - 1; i >= 0; i-- {
+		pathPart := strings.Join(pp[0:i+1], string(os.PathSeparator))
+		if drepseg, has := m.dirReplacementsRendered[pathPart]; has {
+			pp[i] = drepseg
+		}
+	}
+	return strings.Join(pp, string(os.PathSeparator))
+}