feat(modules): allow opting out of build staging (#5890)

Here, we make the `local` config field (previously only available to
`exec` modules) available for all module types.

When used, it maps to the `buildAtSource` field of any generated Build
actions in the module conversion process.

This is done because some users/projects need to reference symlinked
directories in their builds, where the simplest solution appears to be
to simply opt out of build staging for those modules.

core/src/build-staging/build-staging.ts

@@ -132,8 +132,8 @@ export class BuildStaging {
   }
 
   getBuildPath(config: BuildActionConfig<string, any> | ModuleConfig): string {
-    // We don't stage the build for local exec modules, so the module path is effectively the build path.
-    if (config.kind === "Module" && config.type === "exec" && config["local"] === true) {
+    // We don't stage the build for local modules, so the module path is effectively the build path.
+    if (config.kind === "Module" && config["local"] === true) {
       return config.path
     }
 

core/src/config/base.ts

@@ -446,6 +446,7 @@ export function prepareModuleResource(spec: any, configPath: string, projectRoot
       dependencies,
       timeout: spec.build?.timeout || DEFAULT_BUILD_TIMEOUT_SEC,
     },
+    local: spec.local,
     configPath,
     description: spec.description,
     disabled: spec.disabled,

core/src/config/module.ts

@@ -92,6 +92,7 @@ interface ModuleSpecCommon {
   apiVersion?: string
   allowPublish?: boolean
   build?: BaseBuildSpec
+  local?: boolean
   description?: string
   disabled?: boolean
   exclude?: string[]
@@ -187,6 +188,24 @@ export const coreModuleSpecSchema = createSchema({
 // These fields may be resolved later in the process, and allow for usage of template strings
 export const baseModuleSpecKeys = memoize(() => ({
   build: baseBuildSpecSchema().unknown(true),
+  local: joi
+    .boolean()
+    .description(
+      dedent`
+      If set to true, Garden will run the build command, services, tests, and tasks in the module source directory,
+      instead of in the Garden build directory (under .garden/build/<module-name>).
+
+      Garden will therefore not stage the build for local modules. This means that include/exclude filters
+      and ignore files are not applied to local modules, except to calculate the module/action versions.
+
+      If you use use \`build.dependencies[].copy\` for one or more build dependencies of this module, the copied files
+      will be copied to the module source directory (instead of the build directory, as is the default case when
+      \`local = false\`).
+
+      Note: This maps to the \`buildAtSource\` option in this module's generated Build action (if any).
+      `
+    )
+    .default(false),
   description: joi.string().description("A description of the module."),
   disabled: joi
     .boolean()

core/src/plugins/exec/convert.ts

@@ -32,7 +32,6 @@ export function prepareExecBuildAction(params: ConvertModuleParams<ExecModule>):
       ...params.baseFields,
       ...dummyBuild,
 
-      buildAtSource: module.spec.local,
       dependencies: module.build.dependencies.map(convertBuildDependency),
 
       timeout: module.build.timeout,

core/src/plugins/exec/moduleConfig.ts

@@ -32,7 +32,6 @@ import { baseTaskSpecSchema } from "../../config/task.js"
 import { dedent } from "../../util/string.js"
 import type { ExecSyncModeSpec } from "./config.js"
 import type { ConfigureModuleParams, ConfigureModuleResult } from "../../plugin/handlers/Module/configure.js"
-import { ConfigurationError } from "../../exceptions.js"
 import { memoize, omit } from "lodash-es"
 import { DEFAULT_RUN_TIMEOUT_SEC } from "../../constants.js"
 
@@ -59,21 +58,6 @@ const artifactsSchema = memoize(() => joiSparseArray(artifactSchema()))
 export async function configureExecModule({
   moduleConfig,
 }: ConfigureModuleParams<ExecModule>): Promise<ConfigureModuleResult> {
-  const buildDeps = moduleConfig.build.dependencies
-  if (moduleConfig.spec.local && buildDeps.some((d) => d.copy.length > 0)) {
-    const buildDependenciesWithCopySpec = buildDeps
-      .filter((d) => !!d.copy)
-      .map((d) => d.name)
-      .join(", ")
-    throw new ConfigurationError({
-      message: dedent`
-      Invalid exec module configuration: Module ${moduleConfig.name} copies ${buildDependenciesWithCopySpec}
-
-      A local exec module cannot have a build dependency with a copy spec.
-    `,
-    })
-  }
-
   // All the config keys that affect the build version
   moduleConfig.buildConfig = omit(moduleConfig.spec, ["tasks", "tests", "services"])
 
@@ -257,7 +241,6 @@ export interface ExecModuleSpec extends ModuleSpec {
   services: ExecServiceSpec[]
   tasks: ExecTaskSpec[]
   tests: ExecTestSpec[]
-  local?: boolean
 }
 
 export type ExecModuleConfig = ModuleConfig<ExecModuleSpec>
@@ -282,18 +265,6 @@ export const execModuleSpecSchema = createSchema({
   name: "exec:Module",
   description: "The module specification for an exec module.",
   keys: () => ({
-    local: joi
-      .boolean()
-      .description(
-        dedent`
-        If set to true, Garden will run the build command, services, tests, and tasks in the module source directory,
-        instead of in the Garden build directory (under .garden/build/<module-name>).
-
-        Garden will therefore not stage the build for local exec modules. This means that include/exclude filters
-        and ignore files are not applied to local exec modules.
-        `
-      )
-      .default(false),
     build: execModuleBuildSpecSchema(),
     env: joiEnvVars(),
     services: joiSparseArray(execServiceSchema()).description("A list of services to deploy from this module."),

core/src/resolve-module.ts

@@ -914,6 +914,7 @@ export function makeDummyBuild({
 
     copyFrom,
     source: module.repositoryUrl ? { repository: { url: module.repositoryUrl } } : undefined,
+    buildAtSource: module.local,
 
     allowPublish: module.allowPublish,
     dependencies,
@@ -941,6 +942,9 @@ function inheritModuleToAction(module: GardenModule, action: ActionConfig) {
     action.disabled = true
   }
   action.internal.basePath = module.path
+  if (isBuildActionConfig(action)) {
+    action.buildAtSource = module.local
+  }
   if (module.configPath) {
     action.internal.configFilePath = module.configPath
   }

core/src/types/module.ts

@@ -131,9 +131,8 @@ export async function moduleFromConfig({
   const moduleTypes = await garden.getModuleTypes()
   const compatibleTypes = [config.type, ...getModuleTypeBases(moduleTypes[config.type], moduleTypes).map((t) => t.name)]
 
-  // Special-casing local exec modules, otherwise setting build path as <build dir>/<module name>
-  const buildPath =
-    config.type === "exec" && config.spec.local ? config.path : join(garden.buildStaging.buildDirPath, config.name)
+  // Special-casing local modules, otherwise setting build path as <build dir>/<module name>
+  const buildPath = config.local ? config.path : join(garden.buildStaging.buildDirPath, config.name)
 
   await garden.buildStaging.ensureDir(buildPath)
 

core/test/integ/src/plugins/kubernetes/helm/config.ts

@@ -104,6 +104,7 @@ describe("configureHelmModule", () => {
         dependencies: [],
         timeout: DEFAULT_BUILD_TIMEOUT_SEC,
       },
+      local: false,
       configPath: resolve(ctx.projectRoot, "api", "garden.yml"),
       description: "The API backend for the voting UI",
       disabled: false,

core/test/unit/src/config/base.ts

@@ -274,6 +274,7 @@ describe("loadConfigResources", () => {
       repositoryUrl: undefined,
       allowPublish: undefined,
       build: { dependencies: [], timeout: DEFAULT_BUILD_TIMEOUT_SEC },
+      local: undefined,
       path: modulePathA,
       variables: { msg: "OK" },
       varfile: undefined,
@@ -414,6 +415,7 @@ describe("loadConfigResources", () => {
         repositoryUrl: undefined,
         allowPublish: undefined,
         build: { dependencies: [], timeout: DEFAULT_BUILD_TIMEOUT_SEC },
+        local: undefined,
         path: projectPathMultipleModules,
         serviceConfigs: [],
         spec: {
@@ -454,6 +456,7 @@ describe("loadConfigResources", () => {
           dependencies: [{ name: "module-from-project-config", copy: [] }],
           timeout: DEFAULT_BUILD_TIMEOUT_SEC,
         },
+        local: undefined,
         path: modulePathAMultiple,
         serviceConfigs: [],
         spec: {
@@ -484,6 +487,7 @@ describe("loadConfigResources", () => {
         exclude: undefined,
         repositoryUrl: undefined,
         build: { dependencies: [], timeout: DEFAULT_BUILD_TIMEOUT_SEC },
+        local: undefined,
         path: modulePathAMultiple,
         serviceConfigs: [],
         spec: {

core/test/unit/src/plugins/exec/exec.ts

@@ -17,7 +17,6 @@ import { createActionLog } from "../../../../../src/logger/log-entry.js"
 import { keyBy, omit } from "lodash-es"
 import {
   getDataDir,
-  makeTestModule,
   expectError,
   createProjectConfig,
   TestGarden,
@@ -26,15 +25,13 @@ import {
 } from "../../../../helpers.js"
 import { RunTask } from "../../../../../src/tasks/run.js"
 import { makeTestGarden } from "../../../../helpers.js"
-import type { ModuleConfig } from "../../../../../src/config/module.js"
 import type { ConfigGraph } from "../../../../../src/graph/config-graph.js"
 import fsExtra from "fs-extra"
 const { pathExists, emptyDir, readFile, remove } = fsExtra
 import { TestTask } from "../../../../../src/tasks/test.js"
 import { dedent } from "../../../../../src/util/string.js"
 import { sleep } from "../../../../../src/util/util.js"
 import type { ExecModuleConfig } from "../../../../../src/plugins/exec/moduleConfig.js"
-import { configureExecModule } from "../../../../../src/plugins/exec/moduleConfig.js"
 import { actionFromConfig } from "../../../../../src/graph/actions.js"
 import type { TestAction, TestActionConfig } from "../../../../../src/actions/test.js"
 import type { PluginContext } from "../../../../../src/plugin-context.js"
@@ -226,7 +223,7 @@ describe("exec plugin", () => {
         },
       ])
 
-      expect(moduleLocal.spec.local).to.eql(true)
+      expect(moduleLocal.local).to.eql(true)
       expect(moduleLocal.build.dependencies).to.eql([])
       expect(moduleLocal.spec.build.command).to.eql(["pwd"])
 
@@ -356,28 +353,6 @@ describe("exec plugin", () => {
       expect(await pathExists(join(_garden.artifactsPath, "test-outputs", "test-a.txt"))).to.be.true
     })
 
-    describe("configureExecModule", () => {
-      it("should throw if a local exec module has a build.copy spec", async () => {
-        const moduleConfig = makeTestModule(<Partial<ModuleConfig>>{
-          build: {
-            dependencies: [
-              {
-                name: "foo",
-                copy: [
-                  {
-                    source: ".",
-                    target: ".",
-                  },
-                ],
-              },
-            ],
-          },
-          spec: { local: true },
-        })
-        await expectError(async () => await configureExecModule({ ctx, moduleConfig, log }), "configuration")
-      })
-    })
-
     describe("build", () => {
       it("should run the build command in the action dir if local true", async () => {
         const action = graph.getBuild("module-local")
@@ -1122,8 +1097,8 @@ describe("exec plugin", () => {
               makeModuleConfig<ExecModuleConfig>(garden.projectRoot, {
                 name,
                 type: "exec",
+                local, // <---
                 spec: {
-                  local, // <---
                   build: {
                     command: buildCommand,
                   },

docs/reference/commands.md

@@ -1481,6 +1481,21 @@ providers:
           # Maximum time in seconds to wait for build to finish.
           timeout:
 
+        # If set to true, Garden will run the build command, services, tests, and tasks in the module source
+        # directory,
+        # instead of in the Garden build directory (under .garden/build/<module-name>).
+        #
+        # Garden will therefore not stage the build for local modules. This means that include/exclude filters
+        # and ignore files are not applied to local modules, except to calculate the module/action versions.
+        #
+        # If you use use `build.dependencies[].copy` for one or more build dependencies of this module, the copied
+        # files
+        # will be copied to the module source directory (instead of the build directory, as is the default case when
+        # `local = false`).
+        #
+        # Note: This maps to the `buildAtSource` option in this module's generated Build action (if any).
+        local:
+
         # A description of the module.
         description:
 
@@ -2381,6 +2396,19 @@ moduleConfigs:
       # Maximum time in seconds to wait for build to finish.
       timeout:
 
+    # If set to true, Garden will run the build command, services, tests, and tasks in the module source directory,
+    # instead of in the Garden build directory (under .garden/build/<module-name>).
+    #
+    # Garden will therefore not stage the build for local modules. This means that include/exclude filters
+    # and ignore files are not applied to local modules, except to calculate the module/action versions.
+    #
+    # If you use use `build.dependencies[].copy` for one or more build dependencies of this module, the copied files
+    # will be copied to the module source directory (instead of the build directory, as is the default case when
+    # `local = false`).
+    #
+    # Note: This maps to the `buildAtSource` option in this module's generated Build action (if any).
+    local:
+
     # A description of the module.
     description:
 
@@ -2939,6 +2967,19 @@ modules:
       # Maximum time in seconds to wait for build to finish.
       timeout:
 
+    # If set to true, Garden will run the build command, services, tests, and tasks in the module source directory,
+    # instead of in the Garden build directory (under .garden/build/<module-name>).
+    #
+    # Garden will therefore not stage the build for local modules. This means that include/exclude filters
+    # and ignore files are not applied to local modules, except to calculate the module/action versions.
+    #
+    # If you use use `build.dependencies[].copy` for one or more build dependencies of this module, the copied files
+    # will be copied to the module source directory (instead of the build directory, as is the default case when
+    # `local = false`).
+    #
+    # Note: This maps to the `buildAtSource` option in this module's generated Build action (if any).
+    local:
+
     # A description of the module.
     description:
 

docs/reference/config-template-config.md

@@ -69,6 +69,19 @@ modules:
       # Maximum time in seconds to wait for build to finish.
       timeout: 600
 
+    # If set to true, Garden will run the build command, services, tests, and tasks in the module source directory,
+    # instead of in the Garden build directory (under .garden/build/<module-name>).
+    #
+    # Garden will therefore not stage the build for local modules. This means that include/exclude filters
+    # and ignore files are not applied to local modules, except to calculate the module/action versions.
+    #
+    # If you use use `build.dependencies[].copy` for one or more build dependencies of this module, the copied files
+    # will be copied to the module source directory (instead of the build directory, as is the default case when
+    # `local = false`).
+    #
+    # Note: This maps to the `buildAtSource` option in this module's generated Build action (if any).
+    local: false
+
     # A description of the module.
     description:
 
@@ -365,6 +378,26 @@ Maximum time in seconds to wait for build to finish.
 | -------- | ------- | -------- |
 | `number` | `600`   | No       |
 
+### `modules[].local`
+
+[modules](#modules) > local
+
+If set to true, Garden will run the build command, services, tests, and tasks in the module source directory,
+instead of in the Garden build directory (under .garden/build/<module-name>).
+
+Garden will therefore not stage the build for local modules. This means that include/exclude filters
+and ignore files are not applied to local modules, except to calculate the module/action versions.
+
+If you use use `build.dependencies[].copy` for one or more build dependencies of this module, the copied files
+will be copied to the module source directory (instead of the build directory, as is the default case when
+`local = false`).
+
+Note: This maps to the `buildAtSource` option in this module's generated Build action (if any).
+
+| Type      | Default | Required |
+| --------- | ------- | -------- |
+| `boolean` | `false` | No       |
+
 ### `modules[].description`
 
 [modules](#modules) > description