feat(template): allow escaping template strings for generated files

This addresses an issue where template strings couldn't easily be
escaped, such that Garden _wouldn't_ attempt to resolve them.

From the docs:

Sometimes you may want to pass template strings through when generating
files, instead of having Garden resolve them. This could for example be
handy when templating a Terraform configuration file which uses a
similar templating syntax.

To do this, simply add an additional `$` in front of the template
string, e.g. `$${var.dont-resolve-me}`.

core/src/config/config-context.ts

@@ -43,6 +43,8 @@ export interface ContextResolveOpts {
   allowPartial?: boolean
   // a list of previously resolved paths, used to detect circular references
   stack?: string[]
+  // Unescape escaped template strings
+  unescape?: boolean
 }
 
 export interface ContextResolveParams {

core/src/config/module.ts

@@ -119,7 +119,7 @@ const generatedFileSchema = () =>
         .required()
         .description(
           dedent`
-          POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+          POSIX-style filename to write the resolved file contents to, relative to the path of the module source directory (for remote modules this means the root of the module repository, otherwise the directory of the module configuration).
 
           Note that any existing file with the same name will be overwritten. If the path contains one or more directories, they will be automatically created if missing.
           `

core/src/resolve-module.ts

@@ -403,10 +403,9 @@ export class ModuleResolver {
 
       if (fileSpec.sourcePath) {
         contents = (await readFile(fileSpec.sourcePath)).toString()
-        contents = await resolveTemplateString(contents, configContext)
       }
 
-      const resolvedContents = resolveTemplateString(contents, configContext)
+      const resolvedContents = resolveTemplateString(contents, configContext, { unescape: true })
       const targetDir = resolve(resolvedConfig.path, ...posix.dirname(fileSpec.targetPath).split(posix.sep))
       const targetPath = resolve(resolvedConfig.path, ...fileSpec.targetPath.split(posix.sep))
 

core/src/template-string-parser.pegjs

@@ -10,6 +10,7 @@
   const {
     buildBinaryExpression,
     buildLogicalExpression,
+    escapePrefix,
     getKey,
     getValue,
     isArray,
@@ -30,10 +31,25 @@ TemplateString
   / $(.*) { return text() === "" ? [] : [{ resolved: text() }] }
 
 FormatString
-  = FormatStart op:BlockOperator FormatEnd {
+  = EscapeStart SourceCharacter* FormatEndWithOptional {
+    if (options.unescape) {
+      return text().slice(1)
+    } else {
+      return text()
+    }
+  }
+  / FormatStart op:BlockOperator FormatEnd {
     return { block: op }
   }
-  / FormatStart blockOperator:(ExpressionBlockOperator __)* e:Expression end:FormatEndWithOptional {
+  / pre:FormatStartWithEscape blockOperator:(ExpressionBlockOperator __)* e:Expression end:FormatEndWithOptional {
+      if (pre[0] === escapePrefix) {
+        if (options.unescape) {
+          return text().slice(1)
+        } else {
+          return text()
+        }
+      }
+
       // Any unexpected error is returned immediately. Certain exceptions have special semantics that are caught below.
       if (e && e._error && e._error.type !== missingKeyExceptionType && e._error.type !== passthroughExceptionType) {
         return e
@@ -87,9 +103,16 @@ InvalidFormatString
   	throw new TemplateStringError("Unable to parse as valid template string.")
   }
 
+EscapeStart
+  = "$${" __
+
 FormatStart
   = "${" __
 
+FormatStartWithEscape
+  = EscapeStart
+  / FormatStart
+
 FormatEnd
   = __ "}"
 
@@ -108,7 +131,7 @@ ExpressionBlockOperator
   = "if"
 
 Prefix
-  = !FormatStart (. ! FormatStart)* . { return text() }
+  = !FormatStartWithEscape (. ! FormatStartWithEscape)* . { return text() }
 
 Suffix
   = !FormatEnd (. ! FormatEnd)* . { return text() }
@@ -589,15 +612,3 @@ __
 _
   = __
 
-// Automatic Semicolon Insertion
-
-EOS
-  = __ ";"
-  / _ SingleLineComment? LineTerminatorSequence
-  / _ &"}"
-  / __ EOF
-
-EOF
-  = !.
-
-

core/src/template-string.ts

@@ -27,6 +27,7 @@ export type StringOrStringPromise = Promise<string> | string
 
 const missingKeyExceptionType = "template-string-missing-key"
 const passthroughExceptionType = "template-string-passthrough"
+const escapePrefix = "$${"
 
 class TemplateStringError extends GardenBaseError {
   type = "template-string"
@@ -95,6 +96,8 @@ export function resolveTemplateString(string: string, context: ConfigContext, op
       missingKeyExceptionType,
       passthroughExceptionType,
       allowPartial: !!opts.allowPartial,
+      unescape: !!opts.unescape,
+      escapePrefix,
       optionalSuffix: "}?",
       isPlainObject,
       isPrimitive,

core/src/util/fs.ts

@@ -278,6 +278,9 @@ export async function makeTempDir({ git = false }: { git?: boolean } = {}): Prom
 
   if (git) {
     await exec("git", ["init"], { cwd: tmpDir.path })
+    await writeFile(join(tmpDir.path, "foo"), "bar")
+    await exec("git", ["add", "."], { cwd: tmpDir.path })
+    await exec("git", ["commit", "-m", "first commit"], { cwd: tmpDir.path })
   }
 
   return tmpDir

core/test/unit/src/garden.ts

@@ -46,6 +46,7 @@ import { dedent, deline } from "../../../src/util/string"
 import { ServiceState } from "../../../src/types/service"
 import execa from "execa"
 import { getLinkedSources } from "../../../src/util/ext-source-util"
+import { safeDump } from "js-yaml"
 
 describe("Garden", () => {
   let tmpDir: tmp.DirectoryResult
@@ -2898,6 +2899,154 @@ describe("Garden", () => {
       `)
     })
 
+    it("passes escaped template strings through when rendering a file", async () => {
+      const garden = await makeTestGardenA()
+
+      const targetPath = "targetfile.log"
+
+      garden.setModuleConfigs([
+        {
+          apiVersion: DEFAULT_API_VERSION,
+          name: "module-a",
+          type: "test",
+          allowPublish: false,
+          build: { dependencies: [] },
+          disabled: false,
+          include: [],
+          path: pathFoo,
+          serviceConfigs: [],
+          taskConfigs: [],
+          testConfigs: [],
+          spec: {},
+          generateFiles: [
+            {
+              value: "Project name: ${project.name}, Escaped string: $${var.foo}",
+              targetPath,
+            },
+          ],
+        },
+      ])
+
+      const module = await garden.resolveModule("module-a")
+      const expectedTargetPath = join(module.path, targetPath)
+      const contents = await readFile(expectedTargetPath)
+
+      expect(contents.toString()).to.equal("Project name: test-project-a, Escaped string: ${var.foo}")
+    })
+
+    it("resolves and writes a module file in a remote module", async () => {
+      const garden = await makeTestGarden(pathFoo, {
+        config: {
+          apiVersion: DEFAULT_API_VERSION,
+          kind: "Project",
+          name: "test",
+          path: pathFoo,
+          defaultEnvironment: "default",
+          dotIgnoreFiles: [],
+          environments: [{ name: "default", defaultNamespace, variables: {} }],
+          providers: [{ name: "test-plugin" }],
+          variables: {},
+        },
+      })
+
+      const sourcePath = join(pathFoo, "sourcefile.log")
+      const targetPath = "targetfile.log"
+      await writeFile(sourcePath, "hello ${project.name}")
+
+      const tmpRepo = await makeTempDir({ git: true })
+
+      try {
+        garden.setModuleConfigs([
+          {
+            apiVersion: DEFAULT_API_VERSION,
+            name: "module-a",
+            type: "test",
+            allowPublish: false,
+            build: { dependencies: [] },
+            disabled: false,
+            include: [],
+            path: pathFoo,
+            serviceConfigs: [],
+            taskConfigs: [],
+            testConfigs: [],
+            spec: {},
+            repositoryUrl: "file://" + tmpRepo.path + "#master",
+            generateFiles: [
+              {
+                sourcePath,
+                targetPath,
+              },
+            ],
+          },
+        ])
+
+        const module = await garden.resolveModule("module-a")
+
+        // Make sure the resolved module path is in the .garden directory because it's a remote module
+        expect(module.path.startsWith(garden.gardenDirPath)).to.be.true
+
+        const expectedTargetPath = join(module.path, targetPath)
+        const contents = await readFile(expectedTargetPath)
+
+        expect(contents.toString()).to.equal("hello test")
+      } finally {
+        await tmpRepo.cleanup()
+      }
+    })
+
+    it("resolves and writes a module file in a module from a remote source", async () => {
+      const sourcePath = join(pathFoo, "sourcefile.log")
+      const targetPath = "targetfile.log"
+      await writeFile(sourcePath, "hello ${project.name}")
+
+      const tmpRepo = await makeTempDir({ git: true })
+
+      const moduleConfig = {
+        kind: "Module",
+        name: "module-a",
+        type: "test",
+        generateFiles: [
+          {
+            sourcePath,
+            targetPath,
+          },
+        ],
+      }
+
+      await writeFile(join(tmpRepo.path, "module-a.garden.yml"), safeDump(moduleConfig))
+      await exec("git", ["add", "."], { cwd: tmpRepo.path })
+      await exec("git", ["commit", "-m", "add module"], { cwd: tmpRepo.path })
+
+      try {
+        const garden = await makeTestGarden(pathFoo, {
+          config: {
+            apiVersion: DEFAULT_API_VERSION,
+            kind: "Project",
+            name: "test",
+            path: pathFoo,
+            defaultEnvironment: "default",
+            dotIgnoreFiles: [],
+            environments: [{ name: "default", defaultNamespace, variables: {} }],
+            providers: [{ name: "test-plugin" }],
+            sources: [{ name: "remote-module", repositoryUrl: "file://" + tmpRepo.path + "#master" }],
+            variables: {},
+          },
+        })
+
+        const module = await garden.resolveModule("module-a")
+
+        // Make sure the resolved module path is in the .garden directory because it's in a remote source
+        expect(module.path.startsWith(garden.gardenDirPath)).to.be.true
+
+        const expectedTargetPath = join(module.path, targetPath)
+        const contents = await readFile(expectedTargetPath)
+
+        expect(contents.toString()).to.equal("hello test")
+      } finally {
+        await tmpRepo.cleanup()
+      }
+    })
+
     it("should throw if a module type is not recognized", async () => {
       const garden = await makeTestGardenA()
       const config = (await garden.getRawModuleConfigs(["module-a"]))[0]

core/test/unit/src/template-string.ts

@@ -58,6 +58,26 @@ describe("resolveTemplateString", async () => {
     expect(res).to.equal("${foo}?")
   })
 
+  it("should support a string literal in a template string as a means to escape it", async () => {
+    const res = resolveTemplateString("${'$'}{bar}", new TestContext({}))
+    expect(res).to.equal("${bar}")
+  })
+
+  it("should pass through a template string with a double $$ prefix", async () => {
+    const res = resolveTemplateString("$${bar}", new TestContext({}))
+    expect(res).to.equal("$${bar}")
+  })
+
+  it("should allow unescaping a template string with a double $$ prefix", async () => {
+    const res = resolveTemplateString("$${bar}", new TestContext({}), { unescape: true })
+    expect(res).to.equal("${bar}")
+  })
+
+  it("should allow mixing normal and escaped strings", async () => {
+    const res = resolveTemplateString("${foo}-and-$${var.nope}", new TestContext({ foo: "yes" }), { unescape: true })
+    expect(res).to.equal("yes-and-${var.nope}")
+  })
+
   it("should interpolate a format string with a prefix", async () => {
     const res = resolveTemplateString("prefix-${some}", new TestContext({ some: "value" }))
     expect(res).to.equal("prefix-value")

docs/reference/commands.md

@@ -1146,7 +1146,9 @@ providers:
         # The module spec, as defined by the provider plugin.
         spec:
 
-            # POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+            # POSIX-style filename to write the resolved file contents to, relative to the path of the module source
+            # directory (for remote modules this means the root of the module repository, otherwise the directory of
+            # the module configuration).
             #
             # Note that any existing file with the same name will be overwritten. If the path contains one or more
             # directories, they will be automatically created if missing.
@@ -1390,7 +1392,9 @@ moduleConfigs:
     # The module spec, as defined by the provider plugin.
     spec:
 
-        # POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+        # POSIX-style filename to write the resolved file contents to, relative to the path of the module source
+        # directory (for remote modules this means the root of the module repository, otherwise the directory of the
+        # module configuration).
         #
         # Note that any existing file with the same name will be overwritten. If the path contains one or more
         # directories, they will be automatically created if missing.
@@ -1841,7 +1845,9 @@ modules:
     # The module spec, as defined by the provider plugin.
     spec:
 
-        # POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+        # POSIX-style filename to write the resolved file contents to, relative to the path of the module source
+        # directory (for remote modules this means the root of the module repository, otherwise the directory of the
+        # module configuration).
         #
         # Note that any existing file with the same name will be overwritten. If the path contains one or more
         # directories, they will be automatically created if missing.

docs/reference/module-template-config.md

@@ -136,7 +136,9 @@ modules:
         # This file may contain template strings, much like any other field in the configuration.
         sourcePath:
 
-        # POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+        # POSIX-style filename to write the resolved file contents to, relative to the path of the module source
+        # directory (for remote modules this means the root of the module repository, otherwise the directory of the
+        # module configuration).
         #
         # Note that any existing file with the same name will be overwritten. If the path contains one or more
         # directories, they will be automatically created if missing.
@@ -442,7 +444,7 @@ This file may contain template strings, much like any other field in the configu
 
 [modules](#modules) > [generateFiles](#modulesgeneratefiles) > targetPath
 
-POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+POSIX-style filename to write the resolved file contents to, relative to the path of the module source directory (for remote modules this means the root of the module repository, otherwise the directory of the module configuration).
 
 Note that any existing file with the same name will be overwritten. If the path contains one or more directories, they will be automatically created if missing.
 

docs/reference/module-types/conftest.md

@@ -114,7 +114,9 @@ generateFiles:
     # This file may contain template strings, much like any other field in the configuration.
     sourcePath:
 
-    # POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+    # POSIX-style filename to write the resolved file contents to, relative to the path of the module source directory
+    # (for remote modules this means the root of the module repository, otherwise the directory of the module
+    # configuration).
     #
     # Note that any existing file with the same name will be overwritten. If the path contains one or more
     # directories, they will be automatically created if missing.
@@ -361,7 +363,7 @@ This file may contain template strings, much like any other field in the configu
 
 [generateFiles](#generatefiles) > targetPath
 
-POSIX-style filename to write the resolved file contents to, relative to the path of the module.
+POSIX-style filename to write the resolved file contents to, relative to the path of the module source directory (for remote modules this means the root of the module repository, otherwise the directory of the module configuration).
 
 Note that any existing file with the same name will be overwritten. If the path contains one or more directories, they will be automatically created if missing.