feat(core): add ModuleTemplates and templated modules

.circleci/config.yml

@@ -725,6 +725,11 @@ workflows:
           name: e2e-tasks
           project: tasks
           requires: [build]
+      - e2e-project:
+          <<: *only-internal-prs
+          name: e2e-templated-k8s-container
+          project: templated-k8s-container
+          requires: [build]
       - e2e-project:
           <<: *only-internal-prs
           name: e2e-vote

core/src/actions.ts

@@ -849,7 +849,7 @@ export class ActionRouter implements TypeGuard {
       ? Object.keys(runtimeContext.envVars).length === 0 && runtimeContext.dependencies.length === 0
       : true
 
-    if (!runtimeContextIsEmpty && (await getRuntimeTemplateReferences(module)).length > 0) {
+    if (!runtimeContextIsEmpty && getRuntimeTemplateReferences(module).length > 0) {
       log.silly(`Resolving runtime template strings for service '${service.name}'`)
 
       const providers = await this.garden.resolveProviders(log)
@@ -863,6 +863,9 @@ export class ActionRouter implements TypeGuard {
         resolvedProviders: providers,
         dependencies: modules,
         runtimeContext,
+        parentName: module.parentName,
+        templateName: module.templateName,
+        inputs: module.inputs,
       })
 
       // Set allowPartial=false to ensure all required strings are resolved.
@@ -908,7 +911,7 @@ export class ActionRouter implements TypeGuard {
     })
 
     // Resolve ${runtime.*} template strings if needed.
-    if (runtimeContext && (await getRuntimeTemplateReferences(module)).length > 0) {
+    if (runtimeContext && getRuntimeTemplateReferences(module).length > 0) {
       log.silly(`Resolving runtime template strings for task '${task.name}'`)
 
       const providers = await this.garden.resolveProviders(log)
@@ -922,6 +925,9 @@ export class ActionRouter implements TypeGuard {
         resolvedProviders: providers,
         dependencies: modules,
         runtimeContext,
+        parentName: module.parentName,
+        templateName: module.templateName,
+        inputs: module.inputs,
       })
 
       // Set allowPartial=false to ensure all required strings are resolved.

core/src/config-store.ts

@@ -135,7 +135,7 @@ export abstract class ConfigStore<T extends object = any> {
 
   private async loadConfig() {
     await this.ensureConfigFile()
-    const config = (await yaml.safeLoad((await readFile(this.configPath)).toString())) || {}
+    const config = yaml.safeLoad((await readFile(this.configPath)).toString()) || {}
 
     this.config = this.validate(config)
   }

core/src/config/base.ts

@@ -19,14 +19,18 @@ import { validateWithPath } from "./validation"
 import { WorkflowResource } from "./workflow"
 import { listDirectory } from "../util/fs"
 import { isConfigFilename } from "../util/fs"
+import { TemplateKind, templateKind, ModuleTemplateResource } from "./module-template"
 
 export interface GardenResource {
   apiVersion: string
   kind: string
   name: string
   path: string
+  configPath?: string
 }
 
+export type ConfigKind = "Module" | "Workflow" | "Project" | TemplateKind
+
 /**
  * Attempts to parse content as YAML, and applies a linter to produce more informative error messages when
  * content is not valid YAML.
@@ -77,8 +81,6 @@ export async function loadConfigResources(
   return resources
 }
 
-export type ConfigKind = "Module" | "Workflow" | "Project"
-
 /**
  * Each YAML document in a garden.yml file defines a project, a module or a workflow.
  */
@@ -109,6 +111,8 @@ function prepareResource({
     return prepareModuleResource(spec, configPath, projectRoot)
   } else if (kind === "Workflow") {
     return prepareWorkflowResource(spec, configPath)
+  } else if (kind === templateKind) {
+    return prepareTemplateResource(spec, configPath)
   } else if (allowInvalid) {
     return spec
   } else if (!kind) {
@@ -160,6 +164,7 @@ export function prepareModuleResource(spec: any, configPath: string, projectRoot
     configPath,
     description: spec.description,
     disabled: spec.disabled,
+    generateFiles: spec.generateFiles,
     include: spec.include,
     exclude: spec.exclude,
     name: spec.name,
@@ -178,7 +183,7 @@ export function prepareModuleResource(spec: any, configPath: string, projectRoot
   validateWithPath({
     config,
     schema: coreModuleSpecSchema(),
-    path: dirname(configPath),
+    path: configPath,
     projectRoot,
     configType: "module",
     ErrorClass: ConfigurationError,
@@ -199,6 +204,18 @@ export function prepareWorkflowResource(spec: any, configPath: string): Workflow
   return spec
 }
 
+export function prepareTemplateResource(spec: any, configPath: string): ModuleTemplateResource {
+  if (!spec.apiVersion) {
+    spec.apiVersion = DEFAULT_API_VERSION
+  }
+
+  spec.kind = templateKind
+  spec.path = dirname(configPath)
+  spec.configPath = configPath
+
+  return spec
+}
+
 export async function findProjectConfig(path: string, allowInvalid = false): Promise<ProjectResource | undefined> {
   let sepCount = path.split(sep).length - 1
 

core/src/config/common.ts

@@ -12,6 +12,7 @@ import { splitLast } from "../util/util"
 import { deline, dedent } from "../util/string"
 import { cloneDeep } from "lodash"
 import { joiPathPlaceholder } from "./validation"
+import { DEFAULT_API_VERSION } from "../constants"
 
 export const objectSpreadKey = "$merge"
 
@@ -483,3 +484,10 @@ export const moduleVersionSchema = () =>
       .description("The version of each of the dependencies of the module."),
     files: fileNamesSchema(),
   })
+
+export const apiVersionSchema = () =>
+  joi
+    .string()
+    .default(DEFAULT_API_VERSION)
+    .valid(DEFAULT_API_VERSION)
+    .description("The schema version of this config (currently not used).")

core/src/config/config-context.ts

@@ -9,7 +9,15 @@
 import Joi from "@hapi/joi"
 import chalk from "chalk"
 import { isString, mapValues } from "lodash"
-import { PrimitiveMap, joiIdentifierMap, joiStringMap, joiPrimitive, DeepPrimitiveMap, joiVariables } from "./common"
+import {
+  PrimitiveMap,
+  joiIdentifierMap,
+  joiStringMap,
+  joiPrimitive,
+  DeepPrimitiveMap,
+  joiVariables,
+  joiIdentifier,
+} from "./common"
 import { Provider, GenericProviderConfig, ProviderMap } from "./provider"
 import { ConfigurationError } from "../exceptions"
 import { resolveTemplateString, TemplateStringMissingKeyError } from "../template-string"
@@ -21,6 +29,7 @@ import { deline, dedent, naturalList } from "../util/string"
 import { getProviderUrl, getModuleTypeUrl } from "../docs/common"
 import { GardenModule } from "../types/module"
 import { isPrimitive } from "util"
+import { templateKind } from "./module-template"
 
 export type ContextKeySegment = string | number
 export type ContextKey = ContextKeySegment[]
@@ -300,6 +309,13 @@ export class DefaultEnvironmentContext extends ConfigContext {
   }
 }
 
+export interface ProjectConfigContextParams {
+  projectName: string
+  artifactsPath: string
+  username?: string
+  secrets: PrimitiveMap
+}
+
 /**
  * This context is available for template strings for all Project config fields (except `name`, `id` and
  * `domain`).
@@ -328,17 +344,7 @@ export class ProjectConfigContext extends DefaultEnvironmentContext {
   )
   public secrets: PrimitiveMap
 
-  constructor({
-    projectName,
-    artifactsPath,
-    username,
-    secrets,
-  }: {
-    projectName: string
-    artifactsPath: string
-    username?: string
-    secrets: PrimitiveMap
-  }) {
+  constructor({ projectName, artifactsPath, username, secrets }: ProjectConfigContextParams) {
     super({ projectName, artifactsPath, username })
     this.secrets = secrets
   }
@@ -741,6 +747,52 @@ class ErrorContext extends ConfigContext {
   }
 }
 
+export class ParentContext extends ConfigContext {
+  @schema(joiIdentifier().description(`The name of the parent module.`))
+  public name: string
+
+  constructor(root: ConfigContext, name: string) {
+    super(root)
+    this.name = name
+  }
+}
+
+export class ModuleTemplateContext extends ConfigContext {
+  @schema(joiIdentifier().description(`The name of the ${templateKind} being resolved.`))
+  public name: string
+
+  constructor(root: ConfigContext, name: string) {
+    super(root)
+    this.name = name
+  }
+}
+
+export class ModuleTemplateConfigContext extends ProjectConfigContext {
+  @schema(ParentContext.getSchema().description(`Information about the templated module being resolved.`))
+  public parent: ParentContext
+
+  @schema(
+    ModuleTemplateContext.getSchema().description(`Information about the template used when generating the module.`)
+  )
+  public template: ModuleTemplateContext
+
+  @schema(
+    joiVariables().description(`The inputs provided when resolving the ${templateKind}.`).meta({
+      keyPlaceholder: "<input-key>",
+    })
+  )
+  public inputs: DeepPrimitiveMap
+
+  constructor(
+    params: { parentName: string; templateName: string; inputs: DeepPrimitiveMap } & ProjectConfigContextParams
+  ) {
+    super(params)
+    this.parent = new ParentContext(this, params.parentName)
+    this.template = new ModuleTemplateContext(this, params.templateName)
+    this.inputs = params.inputs
+  }
+}
+
 /**
  * This context is available for template strings under the `module` key in configuration files.
  * It is a superset of the context available under the `project` key.
@@ -761,12 +813,36 @@ export class ModuleConfigContext extends ProviderConfigContext {
   )
   public runtime: RuntimeConfigContext
 
+  @schema(
+    ParentContext.getSchema().description(
+      `Information about the parent module (if the module is a submodule, e.g. generated in a templated module).`
+    )
+  )
+  public parent?: ParentContext
+
+  @schema(
+    ModuleTemplateContext.getSchema().description(
+      `Information about the ${templateKind} used when generating the module.`
+    )
+  )
+  public template?: ModuleTemplateContext
+
+  @schema(
+    joiVariables().description(`The inputs provided to the module through a ${templateKind}, if applicable.`).meta({
+      keyPlaceholder: "<input-key>",
+    })
+  )
+  public inputs: DeepPrimitiveMap
+
   constructor({
     garden,
     resolvedProviders,
     moduleName,
     dependencies,
     runtimeContext,
+    parentName,
+    templateName,
+    inputs,
   }: {
     garden: Garden
     resolvedProviders: ProviderMap
@@ -775,6 +851,9 @@ export class ModuleConfigContext extends ProviderConfigContext {
     // We only supply this when resolving configuration in dependency order.
     // Otherwise we pass `${runtime.*} template strings through for later resolution.
     runtimeContext?: RuntimeContext
+    parentName: string | undefined
+    templateName: string | undefined
+    inputs: DeepPrimitiveMap | undefined
   }) {
     super(garden, resolvedProviders)
 
@@ -788,6 +867,11 @@ export class ModuleConfigContext extends ProviderConfigContext {
     }
 
     this.runtime = new RuntimeConfigContext(this, runtimeContext)
+    if (parentName && templateName) {
+      this.parent = new ParentContext(this, parentName)
+      this.template = new ModuleTemplateContext(this, templateName)
+    }
+    this.inputs = inputs || {}
   }
 }
 
@@ -811,6 +895,9 @@ export class OutputConfigContext extends ModuleConfigContext {
       resolvedProviders,
       dependencies: modules,
       runtimeContext,
+      parentName: undefined,
+      templateName: undefined,
+      inputs: {},
     })
   }
 }