feat: optional varfiles (#5996)

feat: add optional property for varfiles

Signed-off-by: Manuel Ruck <git@manuelruck.de>
Co-authored-by: Manuel Ruck <git@manuelruck.de>
Co-authored-by: Vladimir Vagaytsev <vladimir.vagaitsev@gmail.com>

core/src/actions/base.ts

@@ -20,6 +20,8 @@ import {
   parseActionReference,
   createSchema,
   unusedApiVersionSchema,
+  joiArray,
+  joiVarfile,
 } from "../config/common.js"
 import { DOCS_BASE_URL } from "../constants.js"
 import { dedent, naturalList, stableStringify } from "../util/string.js"
@@ -225,7 +227,7 @@ export const baseActionConfigSchema = createSchema({
           A map of variables scoped to this particular action. These are resolved before any other parts of the action configuration and take precedence over group-scoped variables (if applicable) and project-scoped variables, in that order. They may reference group-scoped and project-scoped variables, and generally can use any template strings normally allowed when resolving the action.
         `
       ),
-    varfiles: joiSparseArray(joi.posixPath())
+    varfiles: joiArray(joiVarfile())
       .description(
         dedent`
           Specify a list of paths (relative to the directory where the action is defined) to a file containing variables, that we apply on top of the action-level \`variables\` field, and take precedence over group-level variables (if applicable) and project-level variables, in that order.
@@ -236,7 +238,13 @@ export const baseActionConfigSchema = createSchema({
 
           To use different varfiles in different environments, you can template in the environment name to the varfile name, e.g. \`varfile: "my-action.\$\{environment.name\}.env\` (this assumes that the corresponding varfiles exist).
 
-          If a listed varfile cannot be found, it is ignored.
+          If a listed varfile cannot be found, throwing an error.
+          To add optional varfiles, you can use a list item object with a \`path\` and an optional \`optional\` boolean field.
+          \`\`\`yaml
+          varfiles:
+            - path: my-action.env
+              optional: true
+          \`\`\`
         `
       )
       .example("my-action.env")

core/src/actions/types.ts

@@ -8,7 +8,7 @@
 
 import type { ValuesType } from "utility-types"
 import type { ConfigGraph, ResolvedConfigGraph } from "../graph/config-graph.js"
-import type { ActionReference, DeepPrimitiveMap } from "../config/common.js"
+import type { ActionReference, DeepPrimitiveMap, Varfile } from "../config/common.js"
 import type { ModuleVersion, TreeVersion } from "../vcs/vcs.js"
 import type { BuildAction, BuildActionConfig, ExecutedBuildAction, ResolvedBuildAction } from "./build.js"
 import type { DeployAction, DeployActionConfig, ExecutedDeployAction, ResolvedDeployAction } from "./deploy.js"
@@ -91,7 +91,7 @@ export interface BaseActionConfig<K extends ActionKind = ActionKind, T = string,
   // -> Templating with ActionConfigContext allowed
   variables?: DeepPrimitiveMap
   // -> Templating with ActionConfigContext allowed, including in variables defined in the varfiles
-  varfiles?: string[]
+  varfiles?: Varfile[]
 
   // Type-specific
   spec: Spec

core/src/config/base.ts

@@ -551,11 +551,13 @@ export async function loadVarfile({
   configRoot,
   path,
   defaultPath,
+  optional = false,
 }: {
   // project root (when resolving project config) or module root (when resolving module config)
   configRoot: string
   path: string | undefined
   defaultPath: string | undefined
+  optional?: boolean
 }): Promise<PrimitiveMap> {
   if (!path && !defaultPath) {
     throw new ParameterError({
@@ -565,7 +567,7 @@ export async function loadVarfile({
   const resolvedPath = resolve(configRoot, <string>(path || defaultPath))
   const exists = await pathExists(resolvedPath)
 
-  if (!exists && path && path !== defaultPath) {
+  if (!exists && path && path !== defaultPath && !optional) {
     throw new ConfigurationError({
       message: `Could not find varfile at path '${path}'. Absolute path: ${resolvedPath}`,
     })

core/src/config/common.ts

@@ -60,6 +60,13 @@ export interface DeepPrimitiveMap {
   [key: string]: Primitive | DeepPrimitiveMap | Primitive[] | DeepPrimitiveMap[]
 }
 
+export interface VarfileMap {
+  path: string
+  optional?: boolean
+}
+
+export type Varfile = VarfileMap | string
+
 export const includeGuideLink = makeDocsLinkPlain(
   "using-garden/configuration-overview",
   "#including-excluding-files-and-directories"
@@ -845,6 +852,18 @@ export const joiIdentifierMap = memoize((valueSchema: Joi.Schema) =>
     .description("Key/value map. Keys must be valid identifiers.")
 )
 
+export const joiVarfile = memoize(() =>
+  joi
+    .alternatives(
+      joi.posixPath().description("Path to a file containing a path."),
+      joi.object().keys({
+        path: joi.posixPath().required().description("Path to a file containing a path."),
+        optional: joi.boolean().description("Whether the varfile is optional."),
+      })
+    )
+    .description("A path to a file containing variables, or an object with a path and optional flag.")
+)
+
 export const joiVariablesDescription =
   "Keys may contain letters and numbers. Any values are permitted, including arrays and objects of any nesting."
 

core/src/config/group.ts

@@ -10,8 +10,17 @@ import dedent from "dedent"
 import type { ActionConfig } from "../actions/types.js"
 import { baseActionConfigSchema } from "../actions/base.js"
 import { templateStringLiteral } from "../docs/common.js"
-import type { DeepPrimitiveMap } from "./common.js"
-import { createSchema, joi, joiSparseArray, joiUserIdentifier, joiVariables, unusedApiVersionSchema } from "./common.js"
+import type { DeepPrimitiveMap, Varfile } from "./common.js"
+import {
+  createSchema,
+  joi,
+  joiArray,
+  joiSparseArray,
+  joiUserIdentifier,
+  joiVarfile,
+  joiVariables,
+  unusedApiVersionSchema,
+} from "./common.js"
 import { varfileDescription } from "./base.js"
 
 export interface GroupConfig {
@@ -29,7 +38,7 @@ export interface GroupConfig {
 
   // Variables
   variables?: DeepPrimitiveMap
-  varfiles?: string[]
+  varfiles?: Varfile[]
 
   // Actions
   actions: ActionConfig[]
@@ -50,7 +59,7 @@ export const groupConfig = createSchema({
     variables: joiVariables().default(() => undefined).description(dedent`
       A map of variables scoped to the actions in this group. These are resolved before the actions and take precedence over project-scoped variables. They may reference project-scoped variables, and generally use any template strings normally allowed when resolving the action.
     `),
-    varfiles: joiSparseArray(joi.posixPath())
+    varfiles: joiArray(joiVarfile())
       .description(
         dedent`
           Specify a list of paths (relative to the directory where the group is defined) to a file containing variables, that we apply on top of the group-level \`variables\` field. If you specify multiple paths, they are merged in the order specified, i.e. the last one takes precedence over the previous ones.

core/src/graph/actions.ts

@@ -55,7 +55,7 @@ import {
   resolveTemplateStrings,
 } from "../template-string/template-string.js"
 import { dedent, deline, naturalList } from "../util/string.js"
-import { mergeVariables } from "./common.js"
+import { getVarfileData, mergeVariables } from "./common.js"
 import type { ConfigGraph } from "./config-graph.js"
 import { MutableConfigGraph } from "./config-graph.js"
 import type { ModuleGraph } from "./modules.js"
@@ -627,7 +627,7 @@ export const preprocessActionConfig = profileAsync(async function preprocessActi
   const templateName = config.internal.templateName
 
   // in pre-processing, only use varfiles that are not template strings
-  const resolvedVarFiles = config.varfiles?.filter((f) => !maybeTemplateString(f))
+  const resolvedVarFiles = config.varfiles?.filter((f) => !maybeTemplateString(getVarfileData(f).path))
   const variables = await mergeVariables({
     basePath: config.internal.basePath,
     variables: config.variables,

core/src/graph/common.ts

@@ -15,7 +15,7 @@ import { Profile, profileAsync } from "../util/profiling.js"
 import type { ModuleDependencyGraphNode, ModuleDependencyGraphNodeKind, ModuleGraphNodes } from "./modules.js"
 import type { ActionKind } from "../plugin/action-types.js"
 import { loadVarfile } from "../config/base.js"
-import type { DeepPrimitiveMap } from "../config/common.js"
+import type { DeepPrimitiveMap, Varfile } from "../config/common.js"
 import type { Task } from "../tasks/base.js"
 import type { LogMetadata, TaskLogStatus } from "../logger/log-entry.js"
 
@@ -121,21 +121,29 @@ interface CycleGraph {
   }
 }
 
+export const getVarfileData = (varfile: Varfile) => {
+  const path = typeof varfile === "string" ? varfile : varfile.path
+  const optional = typeof varfile === "string" ? false : varfile.optional
+  return { path, optional }
+}
+
 export const mergeVariables = profileAsync(async function mergeVariables({
   basePath,
   variables,
   varfiles,
 }: {
   basePath: string
   variables?: DeepPrimitiveMap
-  varfiles?: string[]
+  varfiles?: Varfile[]
 }) {
   const varsByFile = await Promise.all(
-    (varfiles || []).map((path) => {
+    (varfiles || []).map((varfile) => {
+      const { path, optional } = getVarfileData(varfile)
       return loadVarfile({
         configRoot: basePath,
         path,
         defaultPath: undefined,
+        optional,
       })
     })
   )

core/test/unit/src/actions/action-configs-to-graph.ts

@@ -557,6 +557,40 @@ describe("actionConfigsToGraph", () => {
     expect(vars).to.eql({ projectName: "${project.name}" })
   })
 
+  it("loads optional varfiles for the action", async () => {
+    const varfilePath = join(tmpDir.path, "varfile.yml")
+    await dumpYaml(varfilePath, {
+      projectName: "${project.name}",
+    })
+
+    const graph = await actionConfigsToGraph({
+      garden,
+      log,
+      groupConfigs: [],
+      configs: [
+        {
+          kind: "Build",
+          type: "test",
+          name: "foo",
+          timeout: DEFAULT_BUILD_TIMEOUT_SEC,
+          varfiles: [{ path: varfilePath, optional: true }],
+          internal: {
+            basePath: tmpDir.path,
+          },
+          spec: {},
+        },
+      ],
+      moduleGraph: new ModuleGraph([], {}),
+      actionModes: {},
+      linkedSources: {},
+    })
+
+    const action = graph.getBuild("foo")
+    const vars = action["variables"]
+
+    expect(vars).to.eql({ projectName: "${project.name}" })
+  })
+
   it("correctly merges varfile with variables", async () => {
     const varfilePath = join(tmpDir.path, "varfile.yml")
     await dumpYaml(varfilePath, {

docs/reference/action-types/Build/container.md

@@ -160,11 +160,17 @@ _NOTE: The default varfile format will change to YAML in Garden v0.13, since YAM
 
 To use different varfiles in different environments, you can template in the environment name to the varfile name, e.g. `varfile: "my-action.\$\{environment.name\}.env` (this assumes that the corresponding varfiles exist).
 
-If a listed varfile cannot be found, it is ignored.
+If a listed varfile cannot be found, throwing an error.
+To add optional varfiles, you can use a list item object with a `path` and an optional `optional` boolean field.
+```yaml
+varfiles:
+  - path: my-action.env
+    optional: true
+```
 
-| Type               | Default | Required |
-| ------------------ | ------- | -------- |
-| `array[posixPath]` | `[]`    | No       |
+| Type                  | Default | Required |
+| --------------------- | ------- | -------- |
+| `array[alternatives]` | `[]`    | No       |
 
 Example:
 
@@ -173,6 +179,26 @@ varfiles:
   "my-action.env"
 ```
 
+### `varfiles[].path`
+
+[varfiles](#varfiles) > path
+
+Path to a file containing a path.
+
+| Type        | Required |
+| ----------- | -------- |
+| `posixPath` | Yes      |
+
+### `varfiles[].optional`
+
+[varfiles](#varfiles) > optional
+
+Whether the varfile is optional.
+
+| Type      | Required |
+| --------- | -------- |
+| `boolean` | No       |
+
 ### `kind`
 
 | Type     | Allowed Values | Required |

docs/reference/action-types/Build/exec.md

@@ -160,11 +160,17 @@ _NOTE: The default varfile format will change to YAML in Garden v0.13, since YAM
 
 To use different varfiles in different environments, you can template in the environment name to the varfile name, e.g. `varfile: "my-action.\$\{environment.name\}.env` (this assumes that the corresponding varfiles exist).
 
-If a listed varfile cannot be found, it is ignored.
+If a listed varfile cannot be found, throwing an error.
+To add optional varfiles, you can use a list item object with a `path` and an optional `optional` boolean field.
+```yaml
+varfiles:
+  - path: my-action.env
+    optional: true
+```
 
-| Type               | Default | Required |
-| ------------------ | ------- | -------- |
-| `array[posixPath]` | `[]`    | No       |
+| Type                  | Default | Required |
+| --------------------- | ------- | -------- |
+| `array[alternatives]` | `[]`    | No       |
 
 Example:
 
@@ -173,6 +179,26 @@ varfiles:
   "my-action.env"
 ```
 
+### `varfiles[].path`
+
+[varfiles](#varfiles) > path
+
+Path to a file containing a path.
+
+| Type        | Required |
+| ----------- | -------- |
+| `posixPath` | Yes      |
+
+### `varfiles[].optional`
+
+[varfiles](#varfiles) > optional
+
+Whether the varfile is optional.
+
+| Type      | Required |
+| --------- | -------- |
+| `boolean` | No       |
+
 ### `kind`
 
 | Type     | Allowed Values | Required |

docs/reference/action-types/Build/jib-container.md

@@ -172,11 +172,17 @@ _NOTE: The default varfile format will change to YAML in Garden v0.13, since YAM
 
 To use different varfiles in different environments, you can template in the environment name to the varfile name, e.g. `varfile: "my-action.\$\{environment.name\}.env` (this assumes that the corresponding varfiles exist).
 
-If a listed varfile cannot be found, it is ignored.
+If a listed varfile cannot be found, throwing an error.
+To add optional varfiles, you can use a list item object with a `path` and an optional `optional` boolean field.
+```yaml
+varfiles:
+  - path: my-action.env
+    optional: true
+```
 
-| Type               | Default | Required |
-| ------------------ | ------- | -------- |
-| `array[posixPath]` | `[]`    | No       |
+| Type                  | Default | Required |
+| --------------------- | ------- | -------- |
+| `array[alternatives]` | `[]`    | No       |
 
 Example:
 
@@ -185,6 +191,26 @@ varfiles:
   "my-action.env"
 ```
 
+### `varfiles[].path`
+
+[varfiles](#varfiles) > path
+
+Path to a file containing a path.
+
+| Type        | Required |
+| ----------- | -------- |
+| `posixPath` | Yes      |
+
+### `varfiles[].optional`
+
+[varfiles](#varfiles) > optional
+
+Whether the varfile is optional.
+
+| Type      | Required |
+| --------- | -------- |
+| `boolean` | No       |
+
 ### `kind`
 
 | Type     | Allowed Values | Required |