feat(config): allow template strings in project source definition

Requested on Slack, kinda surprised this hadn't come up sooner.
Project sources now support same template strings as providers (aside
from referencing providers, since they're initialized later).

core/src/commands/link/source.ts

@@ -67,10 +67,11 @@ export class LinkSourceCommand extends Command<Args> {
     const sourceType = "project"
 
     const { source: sourceName, path } = args
-    const projectSourceToLink = garden.projectSources.find((src) => src.name === sourceName)
+    const projectSources = garden.getProjectSources()
+    const projectSourceToLink = projectSources.find((src) => src.name === sourceName)
 
     if (!projectSourceToLink) {
-      const availableRemoteSources = garden.projectSources.map((s) => s.name).sort()
+      const availableRemoteSources = projectSources.map((s) => s.name).sort()
 
       throw new ParameterError(
         `Remote source ${chalk.underline(sourceName)} not found in project config.` +

core/src/commands/update-remote/sources.ts

@@ -73,7 +73,8 @@ export async function updateRemoteSources({
 }) {
   const { sources } = args
 
-  const projectSources = garden.projectSources.filter((src) => (sources ? sources.includes(src.name) : true))
+  const projectSources = garden.getProjectSources()
+  const selectedSources = projectSources.filter((src) => (sources ? sources.includes(src.name) : true))
 
   const names = projectSources.map((src) => src.name)
 
@@ -83,15 +84,15 @@ export async function updateRemoteSources({
     throw new ParameterError(
       `Expected source(s) ${chalk.underline(diff.join(","))} to be specified in the project garden.yml config.`,
       {
-        remoteSources: garden.projectSources.map((s) => s.name).sort(),
+        remoteSources: projectSources.map((s) => s.name).sort(),
         input: sources ? sources.sort() : undefined,
       }
     )
   }
 
   // TODO Update remotes in parallel. Currently not possible since updating might
   // trigger a username and password prompt from git.
-  for (const { name, repositoryUrl } of projectSources) {
+  for (const { name, repositoryUrl } of selectedSources) {
     await garden.vcs.updateRemoteSource({
       name,
       url: repositoryUrl,
@@ -103,8 +104,8 @@ export async function updateRemoteSources({
   await pruneRemoteSources({
     gardenDirPath: garden.gardenDirPath,
     type: "project",
-    sources: projectSources,
+    sources: selectedSources,
   })
 
-  return { result: { sources: projectSources } }
+  return { result: { sources: selectedSources } }
 }

core/src/garden.ts

@@ -46,7 +46,7 @@ import { LocalConfigStore, ConfigStore, GlobalConfigStore, LinkedSource } from "
 import { getLinkedSources, ExternalSourceType } from "./util/ext-source-util"
 import { BuildDependencyConfig, ModuleConfig } from "./config/module"
 import { ModuleResolver, moduleResolutionConcurrencyLimit } from "./resolve-module"
-import { OutputConfigContext, DefaultEnvironmentContext } from "./config/config-context"
+import { OutputConfigContext, DefaultEnvironmentContext, ProviderConfigContext } from "./config/config-context"
 import { createPluginContext, CommandInfo } from "./plugin-context"
 import { ModuleAndRuntimeActionHandlers, RegisterPluginParam } from "./types/plugin/plugin"
 import { SUPPORTED_PLATFORMS, SupportedPlatform, DEFAULT_GARDEN_DIR_NAME, gardenEnv } from "./constants"
@@ -77,7 +77,7 @@ import { ensureConnected } from "./db/connection"
 import { DependencyValidationGraph } from "./util/validate-dependencies"
 import { Profile } from "./util/profiling"
 import username from "username"
-import { throwOnMissingSecretKeys, resolveTemplateString } from "./template-string"
+import { throwOnMissingSecretKeys, resolveTemplateString, resolveTemplateStrings } from "./template-string"
 import { WorkflowConfig, WorkflowConfigMap, resolveWorkflowConfig } from "./config/workflow"
 import { enterpriseInit } from "./enterprise/init"
 import { PluginTool, PluginTools } from "./util/ext-tools"
@@ -193,7 +193,7 @@ export class Garden {
   public readonly namespace: string
   public readonly variables: DeepPrimitiveMap
   public readonly secrets: StringMap
-  public readonly projectSources: SourceConfig[]
+  private readonly projectSources: SourceConfig[]
   public readonly buildStaging: BuildStaging
   public readonly gardenDirPath: string
   public readonly artifactsPath: string
@@ -997,7 +997,8 @@ export class Garden {
       // Add external sources that are defined at the project level. External sources are either kept in
       // the .garden/sources dir (and cloned there if needed), or they're linked to a local path via the link command.
       const linkedSources = await getLinkedSources(this, "project")
-      const extSourcePaths = await Bluebird.map(this.projectSources, ({ name, repositoryUrl }) => {
+      const projectSources = this.getProjectSources()
+      const extSourcePaths = await Bluebird.map(projectSources, ({ name, repositoryUrl }) => {
         return this.loadExtSourcePath({
           name,
           linkedSources,
@@ -1127,6 +1128,14 @@ export class Garden {
   //region Internal helpers
   //===========================================================================
 
+  /**
+   * Returns the configured project sources, and resolves any template strings on them.
+   */
+  public getProjectSources() {
+    const context = new ProviderConfigContext(this, {})
+    return resolveTemplateStrings(this.projectSources, context)
+  }
+
   /**
    * Clones the project/module source if needed and returns the path (either from .garden/sources or from a local path)
    */

core/src/plugin-context.ts

@@ -7,8 +7,7 @@
  */
 
 import { Garden } from "./garden"
-import { cloneDeep } from "lodash"
-import { projectNameSchema, projectSourcesSchema, environmentNameSchema } from "./config/project"
+import { projectNameSchema, projectSourcesSchema, environmentNameSchema, SourceConfig } from "./config/project"
 import { Provider, providerSchema, GenericProviderConfig } from "./config/provider"
 import { deline } from "./util/string"
 import { joi, joiVariables, PrimitiveMap, joiStringMap } from "./config/common"
@@ -18,7 +17,6 @@ type WrappedFromGarden = Pick<
   Garden,
   | "projectName"
   | "projectRoot"
-  | "projectSources"
   | "gardenDirPath"
   | "workingCopyId"
   // TODO: remove this from the interface
@@ -34,6 +32,7 @@ export interface CommandInfo {
 
 export interface PluginContext<C extends GenericProviderConfig = GenericProviderConfig> extends WrappedFromGarden {
   command?: CommandInfo
+  projectSources: SourceConfig[]
   provider: Provider<C>
   tools: { [key: string]: PluginTool }
 }
@@ -83,7 +82,7 @@ export async function createPluginContext(
     gardenDirPath: garden.gardenDirPath,
     projectName: garden.projectName,
     projectRoot: garden.projectRoot,
-    projectSources: cloneDeep(garden.projectSources),
+    projectSources: garden.getProjectSources(),
     provider,
     production: garden.production,
     workingCopyId: garden.workingCopyId,

core/test/unit/src/commands/get/get-linked-repos.ts

@@ -30,7 +30,7 @@ describe("GetLinkedReposCommand", () => {
   it("should list all linked project sources in the project", async () => {
     garden = await makeExtProjectSourcesGarden()
     const log = garden.log
-    const sourcesDir = getDataDir("test-project-local-module-sources")
+    const sourcesDir = getDataDir("test-project-local-project-sources")
     const linkSourceCmd = new LinkSourceCommand()
     const sourceNames = ["source-a", "source-b", "source-c"]
     for (const sourceName of sourceNames) {
@@ -64,7 +64,7 @@ describe("GetLinkedReposCommand", () => {
   it("should list all linked modules in the project", async () => {
     garden = await makeExtModuleSourcesGarden()
     const log = garden.log
-    const sourcesDir = getDataDir("test-project-local-project-sources")
+    const sourcesDir = getDataDir("test-project-local-module-sources")
     const linkModuleCmd = new LinkModuleCommand()
     const sourceNames = ["module-a", "module-b", "module-c"]
     for (const moduleName of sourceNames) {

core/test/unit/src/garden.ts

@@ -27,7 +27,7 @@ import {
   resetLocalConfig,
   testGitUrl,
 } from "../../helpers"
-import { getNames, findByName, omitUndefined } from "../../../src/util/util"
+import { getNames, findByName, omitUndefined, exec } from "../../../src/util/util"
 import { LinkedSource } from "../../../src/config-store"
 import { ModuleVersion } from "../../../src/vcs/vcs"
 import { getModuleCacheContext } from "../../../src/types/module"
@@ -40,8 +40,8 @@ import { providerConfigBaseSchema } from "../../../src/config/provider"
 import { keyBy, set, mapValues } from "lodash"
 import stripAnsi from "strip-ansi"
 import { joi } from "../../../src/config/common"
-import { defaultDotIgnoreFiles } from "../../../src/util/fs"
-import { realpath, writeFile, readFile, remove } from "fs-extra"
+import { defaultDotIgnoreFiles, makeTempDir } from "../../../src/util/fs"
+import { realpath, writeFile, readFile, remove, pathExists, mkdirp, copy } from "fs-extra"
 import { dedent, deline } from "../../../src/util/string"
 import { ServiceState } from "../../../src/types/service"
 import execa from "execa"
@@ -2258,6 +2258,46 @@ describe("Garden", () => {
       expect(getNames(modules).sort()).to.eql(["module-a", "module-b", "module-c"])
     })
 
+    it("should resolve template strings in project source definitions", async () => {
+      const garden = await makeTestGarden(resolve(dataDir, "test-project-ext-project-sources"))
+      const sourcesPath = join(garden.gardenDirPath, "sources")
+
+      if (await pathExists(sourcesPath)) {
+        await remove(sourcesPath)
+        await mkdirp(sourcesPath)
+      }
+
+      const localSourcePath = resolve(dataDir, "test-project-local-project-sources", "source-a")
+      const _tmpDir = await makeTempDir()
+
+      try {
+        // Create a temporary git repo to clone
+        const repoPath = resolve(_tmpDir.path, garden.projectName)
+        await copy(localSourcePath, repoPath)
+        await exec("git", ["init"], { cwd: repoPath })
+        await exec("git", ["add", "."], { cwd: repoPath })
+        await exec("git", ["commit", "-m", "foo"], { cwd: repoPath })
+
+        garden.variables.sourceBranch = "master"
+
+        const _garden = garden as any
+        _garden["projectSources"] = [
+          {
+            name: "source-a",
+            // Use a couple of template strings in the repo path
+            repositoryUrl: "file://" + _tmpDir.path + "/${project.name}#${var.sourceBranch}",
+          },
+        ]
+
+        await garden.scanAndAddConfigs()
+
+        const modules = await garden.resolveModules({ log: garden.log })
+        expect(getNames(modules).sort()).to.eql(["module-a"])
+      } finally {
+        await _tmpDir.cleanup()
+      }
+    })
+
     it("should resolve module templates and any modules referencing them", async () => {
       const root = resolve(dataDir, "test-projects", "module-templates")
       const garden = await makeTestGarden(root)