diff --git a/dashboard/src/containers/sidebar.tsx b/dashboard/src/containers/sidebar.tsx index cd14da4d50..d01ec349db 100644 --- a/dashboard/src/containers/sidebar.tsx +++ b/dashboard/src/containers/sidebar.tsx @@ -11,7 +11,7 @@ import React, { useContext, useEffect } from "react" import Sidebar from "../components/sidebar" import { DataContext } from "../context/data" -import { DashboardPage } from "garden-service/build/src/config/dashboard" +import { DashboardPage } from "garden-service/build/src/config/status" export interface Page extends DashboardPage { path: string diff --git a/docs/reference/module-types/openfaas.md b/docs/reference/module-types/openfaas.md index 013d322a5e..3b54029723 100644 --- a/docs/reference/module-types/openfaas.md +++ b/docs/reference/module-types/openfaas.md @@ -1,6 +1,7 @@ # `openfaas` reference - +Deploy [OpenFaaS](https://www.openfaas.com/) functions using Garden. Requires either the `openfaas` or +`local-openfaas` provider to be configured. Below is the schema reference. For an introduction to configuring Garden modules, please look at our [Configuration guide](../../using-garden/configuration-files.md). @@ -198,6 +199,104 @@ POSIX-style path or filename to copy the directory or file(s). | -------- | -------- | ------------------------- | | `string` | No | `""` | +### `dependencies` + +The names of services/functions that this function depends on at runtime. + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[string]` | No | `[]` | + +### `env` + +Key/value map of environment variables. Keys must be valid POSIX environment variable names (must not start with `GARDEN`) and values must be primitives. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `object` | No | `{}` | + +### `handler` + +Specify which directory under the module contains the handler file/function. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `string` | No | `"."` | + +### `image` + +The image name to use for the built OpenFaaS container (defaults to the module name) + +| Type | Required | +| -------- | -------- | +| `string` | No | + +### `lang` + +The OpenFaaS language template to use to build this function. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tests` + +A list of tests to run in the module. + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[object]` | No | `[]` | + +### `tests[].name` + +[tests](#tests) > name + +The name of the test. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `tests[].dependencies[]` + +[tests](#tests) > dependencies + +The names of any services that must be running, and the names of any tasks that must be executed, before the test is run. + +| Type | Required | Default | +| --------------- | -------- | ------- | +| `array[string]` | No | `[]` | + +### `tests[].timeout` + +[tests](#tests) > timeout + +Maximum duration (in seconds) of the test run. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `number` | No | `null` | + +### `tests[].command[]` + +[tests](#tests) > command + +The command to run in the module build context in order to test it. + +| Type | Required | +| --------------- | -------- | +| `array[string]` | No | + +### `tests[].env` + +[tests](#tests) > env + +Key/value map of environment variables. Keys must be valid POSIX environment variable names (must not start with `GARDEN`) and values must be primitives. + +| Type | Required | Default | +| -------- | -------- | ------- | +| `object` | No | `{}` | + ## Complete YAML schema ```yaml @@ -216,6 +315,17 @@ build: copy: - source: target: +dependencies: [] +env: {} +handler: . +image: +lang: +tests: + - name: + dependencies: [] + timeout: null + command: + env: {} ``` ## Outputs @@ -272,3 +382,13 @@ The outputs defined by the module. | Type | Required | | -------- | -------- | | `object` | Yes | + +### `modules..outputs.endpoint` + +[outputs](#outputs) > endpoint + +The full URL to query this service _from within_ the cluster. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | diff --git a/docs/reference/providers/kubernetes.md b/docs/reference/providers/kubernetes.md index d1b75ae601..54152b1c47 100644 --- a/docs/reference/providers/kubernetes.md +++ b/docs/reference/providers/kubernetes.md @@ -928,3 +928,24 @@ providers: namespace: setupIngressController: false ``` + +## Outputs + +The following keys are available via the `${providers.}` template string key for `kubernetes` +providers. + +### `providers..app-namespace` + +The primary namespace used for resource deployments. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `providers..metadata-namespace` + +The namespace used for Garden metadata. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | diff --git a/docs/reference/providers/local-kubernetes.md b/docs/reference/providers/local-kubernetes.md index 4ced663f3f..908616921c 100644 --- a/docs/reference/providers/local-kubernetes.md +++ b/docs/reference/providers/local-kubernetes.md @@ -832,3 +832,24 @@ providers: namespace: setupIngressController: nginx ``` + +## Outputs + +The following keys are available via the `${providers.}` template string key for `local-kubernetes` +providers. + +### `providers..app-namespace` + +The primary namespace used for resource deployments. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | + +### `providers..metadata-namespace` + +The namespace used for Garden metadata. + +| Type | Required | +| -------- | -------- | +| `string` | Yes | diff --git a/docs/reference/providers/maven-container.md b/docs/reference/providers/maven-container.md index 44c879dce1..e023e14a92 100644 --- a/docs/reference/providers/maven-container.md +++ b/docs/reference/providers/maven-container.md @@ -12,40 +12,40 @@ The reference is divided into two sections. The [first section](#configuration-k | --------------- | -------- | ------- | | `array[object]` | No | `[]` | -### `providers[].environments[]` +### `providers[].name` -[providers](#providers) > environments +[providers](#providers) > name -If specified, this provider will only be used in the listed environments. Note that an empty array effectively disables the provider. To use a provider in all environments, omit this field. +The name of the provider plugin to use. -| Type | Required | -| --------------- | -------- | -| `array[string]` | No | +| Type | Required | +| -------- | -------- | +| `string` | Yes | Example: ```yaml providers: - - environments: - - dev - - stage + - name: "local-kubernetes" ``` -### `providers[].name` +### `providers[].environments[]` -[providers](#providers) > name +[providers](#providers) > environments -The name of the provider plugin to use. +If specified, this provider will only be used in the listed environments. Note that an empty array effectively disables the provider. To use a provider in all environments, omit this field. -| Type | Required | Default | -| -------- | -------- | ------------------- | -| `string` | Yes | `"maven-container"` | +| Type | Required | +| --------------- | -------- | +| `array[string]` | No | Example: ```yaml providers: - - name: "maven-container" + - environments: + - dev + - stage ``` @@ -55,6 +55,6 @@ The values in the schema below are the default values. ```yaml providers: - - environments: - name: maven-container + - name: + environments: ``` diff --git a/garden-service/src/actions.ts b/garden-service/src/actions.ts index a13ef97819..dde7207038 100644 --- a/garden-service/src/actions.ts +++ b/garden-service/src/actions.ts @@ -9,12 +9,12 @@ import Bluebird = require("bluebird") import chalk from "chalk" -import { fromPairs, keyBy, mapValues, omit, pickBy, values } from "lodash" +import { fromPairs, keyBy, mapValues, omit, pickBy } from "lodash" import { PublishModuleParams, PublishResult } from "./types/plugin/module/publishModule" import { SetSecretParams, SetSecretResult } from "./types/plugin/provider/setSecret" import { validate, joi } from "./config/common" -import { defaultProvider, Provider } from "./config/provider" +import { defaultProvider } from "./config/provider" import { ParameterError, PluginError } from "./exceptions" import { ActionHandlerMap, Garden, ModuleActionHandlerMap, ModuleActionMap, PluginActionMap } from "./garden" import { LogEntry } from "./logger/log-entry" @@ -52,10 +52,15 @@ import { moduleActionNames, pluginActionDescriptions, pluginActionNames, + GardenPlugin, } from "./types/plugin/plugin" import { CleanupEnvironmentParams } from "./types/plugin/provider/cleanupEnvironment" import { DeleteSecretParams, DeleteSecretResult } from "./types/plugin/provider/deleteSecret" -import { EnvironmentStatusMap, GetEnvironmentStatusParams } from "./types/plugin/provider/getEnvironmentStatus" +import { + EnvironmentStatusMap, + GetEnvironmentStatusParams, + EnvironmentStatus, +} from "./types/plugin/provider/getEnvironmentStatus" import { GetSecretParams, GetSecretResult } from "./types/plugin/provider/getSecret" import { DeleteServiceParams } from "./types/plugin/service/deleteService" import { DeployServiceParams } from "./types/plugin/service/deployService" @@ -69,6 +74,7 @@ import { RunTaskParams, RunTaskResult } from "./types/plugin/task/runTask" import { Service, ServiceStatus, ServiceStatusMap, getServiceRuntimeContext } from "./types/service" import { Omit } from "./util/util" import { DebugInfoMap } from "./types/plugin/provider/getDebugInfo" +import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "./types/plugin/provider/prepareEnvironment" type TypeGuard = { readonly [P in keyof (PluginActionParams | ModuleActionParams)]: (...args: any[]) => Promise @@ -108,20 +114,16 @@ export class ActionHelper implements TypeGuard { private readonly actionHandlers: PluginActionMap private readonly moduleActionHandlers: ModuleActionMap - constructor( - private garden: Garden, - providers: Provider[], - ) { + constructor(private garden: Garden, plugins: { [key: string]: GardenPlugin }) { this.actionHandlers = fromPairs(pluginActionNames.map(n => [n, {}])) this.moduleActionHandlers = fromPairs(moduleActionNames.map(n => [n, {}])) - for (const provider of providers) { - const plugin = garden.getPlugin(provider.name) + for (const [name, plugin] of Object.entries(plugins)) { const actions = plugin.actions || {} for (const actionType of pluginActionNames) { const handler = actions[actionType] - handler && this.addActionHandler(provider.name, actionType, handler) + handler && this.addActionHandler(name, plugin, actionType, handler) } const moduleActions = plugin.moduleActions || {} @@ -129,7 +131,7 @@ export class ActionHelper implements TypeGuard { for (const moduleType of Object.keys(moduleActions)) { for (const actionType of moduleActionNames) { const handler = moduleActions[moduleType][actionType] - handler && this.addModuleActionHandler(provider.name, actionType, moduleType, handler) + handler && this.addModuleActionHandler(name, plugin, actionType, moduleType, handler) } } } @@ -140,77 +142,41 @@ export class ActionHelper implements TypeGuard { //=========================================================================== async getEnvironmentStatus( - { pluginName, log }: ActionHelperParams, - ): Promise { - const handlers = this.getActionHandlers("getEnvironmentStatus", pluginName) - const logEntry = log.debug({ - msg: "Getting status...", - status: "active", - section: `${this.garden.environmentName} environment`, + params: RequirePluginName>, + ): Promise { + const { pluginName } = params + + return this.callActionHandler({ + actionType: "getEnvironmentStatus", + pluginName, + params: omit(params, ["pluginName"]), + defaultHandler: async () => ({ ready: true, outputs: {} }), }) - const res = await Bluebird.props(mapValues(handlers, async (h) => h({ ...await this.commonParams(h, logEntry) }))) - logEntry.setSuccess("Ready") - return res } - /** - * Checks environment status and calls prepareEnvironment for each provider that isn't flagged as ready. - * - * If any of the getEnvironmentStatus handlers return ready=false. - */ async prepareEnvironment( - { force = false, pluginName, log }: - { force?: boolean, pluginName?: string, log: LogEntry }, - ) { - const entry = log.info({ section: "providers", msg: "Getting status...", status: "active" }) - const statuses = await this.getEnvironmentStatus({ pluginName, log: entry }) - - const prepareHandlers = this.getActionHandlers("prepareEnvironment", pluginName) + params: RequirePluginName>, + ): Promise { + const { pluginName } = params - const needPrep = Object.entries(prepareHandlers).filter(([name]) => { - const status = statuses[name] || { ready: false } - return (force || !status.ready) + return this.callActionHandler({ + actionType: "prepareEnvironment", + pluginName, + params: omit(params, ["pluginName"]), + defaultHandler: async () => ({ status: { ready: true, outputs: {} } }), }) - - const output = {} - - if (needPrep.length > 0) { - entry.setState(`Preparing environment...`) - } - - // sequentially go through the preparation steps, to allow plugins to request user input - for (const [name, handler] of needPrep) { - const status = statuses[name] || { ready: false } - - const envLogEntry = entry.info({ - status: "active", - section: name, - msg: "Configuring...", - }) - - await handler({ - ...await this.commonParams(handler, log), - force, - status, - log: envLogEntry, - }) - - envLogEntry.setSuccess({ msg: chalk.green("Ready"), append: true }) - - output[name] = true - } - - entry.setSuccess({ msg: chalk.green("Ready"), append: true }) - - return output } async cleanupEnvironment( - { pluginName, log }: ActionHelperParams, - ): Promise { - const handlers = this.getActionHandlers("cleanupEnvironment", pluginName) - await Bluebird.each(values(handlers), async (h) => h({ ...await this.commonParams(h, log) })) - return this.getEnvironmentStatus({ pluginName, log }) + params: RequirePluginName>, + ) { + const { pluginName } = params + return this.callActionHandler({ + actionType: "cleanupEnvironment", + pluginName, + params: omit(params, ["pluginName"]), + defaultHandler: async () => ({}), + }) } async getSecret(params: RequirePluginName>): Promise { @@ -372,7 +338,7 @@ export class ActionHelper implements TypeGuard { async getStatus({ log, serviceNames }: { log: LogEntry, serviceNames?: string[] }): Promise { log.verbose(`Getting environment status (${this.garden.projectName})`) - const envStatus: EnvironmentStatusMap = await this.getEnvironmentStatus({ log }) + const envStatus = await this.garden.getEnvironmentStatus() const serviceStatuses = await this.getServiceStatuses({ log, serviceNames }) return { providers: envStatus, @@ -445,14 +411,21 @@ export class ActionHelper implements TypeGuard { log.info("") const envLog = log.info({ msg: chalk.white("Cleaning up environments..."), status: "active" }) - const environmentStatuses = await this.cleanupEnvironment({ log: envLog }) + const environmentStatuses: EnvironmentStatusMap = {} + + const providers = await this.garden.resolveProviders() + await Bluebird.each(providers, async (provider) => { + await this.cleanupEnvironment({ pluginName: provider.name, log: envLog }) + environmentStatuses[provider.name] = await this.getEnvironmentStatus({ pluginName: provider.name, log: envLog }) + }) + envLog.setSuccess() return { serviceStatuses, environmentStatuses } } async getDebugInfo({ log, includeProject }: { log: LogEntry, includeProject: boolean }): Promise { - const handlers = this.getActionHandlers("getDebugInfo") + const handlers = await this.getActionHandlers("getDebugInfo") return Bluebird.props(mapValues(handlers, async (h) => h({ ...await this.commonParams(h, log), includeProject }))) } @@ -460,8 +433,9 @@ export class ActionHelper implements TypeGuard { // TODO: find a nicer way to do this (like a type-safe wrapper function) private async commonParams(handler, log: LogEntry): Promise { + const provider = await this.garden.resolveProvider(handler["pluginName"]) return { - ctx: await this.garden.getPluginContext(handler["pluginName"]), + ctx: await this.garden.getPluginContext(provider), // TODO: find a better way for handlers to log during execution log, } @@ -472,11 +446,12 @@ export class ActionHelper implements TypeGuard { { params: ActionHelperParams, actionType: T, - pluginName?: string, + pluginName: string, defaultHandler?: PluginActions[T], }, ): Promise { - const handler = this.getActionHelper({ + this.garden.log.silly(`Calling '${actionType}' handler on '${pluginName}'`) + const handler = await this.getActionHandler({ actionType, pluginName, defaultHandler, @@ -485,15 +460,16 @@ export class ActionHelper implements TypeGuard { ...await this.commonParams(handler, (params).log), ...params, } - return (handler)(handlerParams) + const result = (handler)(handlerParams) + this.garden.log.silly(`Called '${actionType}' handler on ${pluginName}'`) + return result } private async callModuleHandler>( { params, actionType, defaultHandler }: { params: ModuleActionHelperParams, actionType: T, defaultHandler?: ModuleActions[T] }, ): Promise { - // the type system is messing me up here, not sure why I need the any cast... - j.e. - const { module, pluginName, log } = params + const { module, pluginName, log } = params log.verbose(`Getting ${actionType} handler for module ${module.name}`) @@ -504,7 +480,7 @@ export class ActionHelper implements TypeGuard { defaultHandler, }) - const handlerParams: any = { + const handlerParams = { ...await this.commonParams(handler, (params).log), ...params, module: omit(module, ["_ConfigType"]), @@ -520,7 +496,7 @@ export class ActionHelper implements TypeGuard { { params, actionType, defaultHandler }: { params: ServiceActionHelperParams, actionType: T, defaultHandler?: ServiceActions[T] }, ): Promise { - const { log, service, runtimeContext } = params + const { log, service, runtimeContext } = params const module = service.module log.verbose(`Getting ${actionType} handler for service ${service.name}`) @@ -532,7 +508,7 @@ export class ActionHelper implements TypeGuard { defaultHandler, }) - const handlerParams: any = { + const handlerParams = { ...await this.commonParams(handler, log), ...params, module, @@ -577,13 +553,19 @@ export class ActionHelper implements TypeGuard { } private addActionHandler( - pluginName: string, actionType: T, handler: PluginActions[T], + pluginName: string, plugin: GardenPlugin, actionType: T, handler: PluginActions[T], ) { - const plugin = this.garden.getPlugin(pluginName) const schema = pluginActionDescriptions[actionType].resultSchema const wrapped = async (...args) => { const result = await handler.apply(plugin, args) + if (result === undefined) { + throw new PluginError(`Got empty response from ${actionType} handler on ${pluginName}`, { + args, + actionType, + pluginName, + }) + } return validate(result, schema, { context: `${actionType} output from plugin ${pluginName}` }) } wrapped["actionType"] = actionType @@ -593,13 +575,19 @@ export class ActionHelper implements TypeGuard { } private addModuleActionHandler( - pluginName: string, actionType: T, moduleType: string, handler: ModuleActions[T], + pluginName: string, plugin: GardenPlugin, actionType: T, moduleType: string, handler: ModuleActions[T], ) { - const plugin = this.garden.getPlugin(pluginName) const schema = moduleActionDescriptions[actionType].resultSchema const wrapped = async (...args: any[]) => { const result = await handler.apply(plugin, args) + if (result === undefined) { + throw new PluginError(`Got empty response from ${moduleType}.${actionType} handler on ${pluginName}`, { + args, + actionType, + pluginName, + }) + } return validate(result, schema, { context: `${actionType} output from plugin ${pluginName}` }) } wrapped["actionType"] = actionType @@ -620,24 +608,26 @@ export class ActionHelper implements TypeGuard { /** * Get a handler for the specified action. */ - public getActionHandlers(actionType: T, pluginName?: string): ActionHandlerMap { + public async getActionHandlers( + actionType: T, pluginName?: string, + ): Promise> { return this.filterActionHandlers(this.actionHandlers[actionType], pluginName) } /** * Get a handler for the specified module action. */ - public getModuleActionHandlers( + public async getModuleActionHandlers( { actionType, moduleType, pluginName }: { actionType: T, moduleType: string, pluginName?: string }, - ): ModuleActionHandlerMap { + ): Promise> { return this.filterActionHandlers((this.moduleActionHandlers[actionType] || {})[moduleType], pluginName) } - private filterActionHandlers(handlers, pluginName?: string) { + private async filterActionHandlers(handlers, pluginName?: string) { // make sure plugin is loaded if (!!pluginName) { - this.garden.getPlugin(pluginName) + await this.garden.getPlugin(pluginName) } if (handlers === undefined) { @@ -650,17 +640,19 @@ export class ActionHelper implements TypeGuard { /** * Get the last configured handler for the specified action (and optionally module type). */ - public getActionHelper( + public async getActionHandler( { actionType, pluginName, defaultHandler }: - { actionType: T, pluginName?: string, defaultHandler?: PluginActions[T] }, - ): PluginActions[T] { + { actionType: T, pluginName: string, defaultHandler?: PluginActions[T] }, + ): Promise { - const handlers = Object.values(this.getActionHandlers(actionType, pluginName)) + const handlers = Object.values(await this.getActionHandlers(actionType, pluginName)) if (handlers.length) { + this.garden.log.silly(`Found '${actionType}' handler on '${pluginName}'`) return handlers[handlers.length - 1] } else if (defaultHandler) { defaultHandler["pluginName"] = defaultProvider.name + this.garden.log.silly(`Returned default '${actionType}' handler for '${pluginName}'`) return defaultHandler } @@ -684,12 +676,12 @@ export class ActionHelper implements TypeGuard { /** * Get the last configured handler for the specified action. */ - public getModuleActionHandler( + public async getModuleActionHandler( { actionType, moduleType, pluginName, defaultHandler }: { actionType: T, moduleType: string, pluginName?: string, defaultHandler?: ModuleAndRuntimeActions[T] }, - ): ModuleAndRuntimeActions[T] { + ): Promise { - const handlers = Object.values(this.getModuleActionHandlers({ actionType, moduleType, pluginName })) + const handlers = Object.values(await this.getModuleActionHandlers({ actionType, moduleType, pluginName })) if (handlers.length) { return handlers[handlers.length - 1] diff --git a/garden-service/src/commands/deploy.ts b/garden-service/src/commands/deploy.ts index 791dedc731..faae32fe09 100644 --- a/garden-service/src/commands/deploy.ts +++ b/garden-service/src/commands/deploy.ts @@ -120,10 +120,6 @@ export class DeployCommand extends Command { watch = opts.watch } - // TODO: make this a task - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) - const results = await processServices({ garden, graph: initGraph, diff --git a/garden-service/src/commands/dev.ts b/garden-service/src/commands/dev.ts index afd92ec12c..f11ff21a96 100644 --- a/garden-service/src/commands/dev.ts +++ b/garden-service/src/commands/dev.ts @@ -101,7 +101,6 @@ export class DevCommand extends Command { async action({ garden, log, footerLog, opts }: CommandParams): Promise { this.server.setGarden(garden) - const actions = await garden.getActionHelper() const graph = await garden.getConfigGraph() const modules = await graph.getModules() @@ -121,8 +120,6 @@ export class DevCommand extends Command { } } - await actions.prepareEnvironment({ log }) - const tasksForModule = (watch: boolean) => { return async (updatedGraph: ConfigGraph, module: Module) => { const tasks: BaseTask[] = [] diff --git a/garden-service/src/commands/init.ts b/garden-service/src/commands/init.ts index e6b88a8d7c..9e688da0bd 100644 --- a/garden-service/src/commands/init.ts +++ b/garden-service/src/commands/init.ts @@ -40,12 +40,11 @@ export class InitCommand extends Command { options = initOpts - async action({ garden, log, footerLog, headerLog, opts }: CommandParams<{}, Opts>): Promise> { + async action({ garden, footerLog, headerLog, opts }: CommandParams<{}, Opts>): Promise> { const name = garden.environmentName printHeader(headerLog, `Initializing ${name} environment`, "gear") - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log, force: opts.force }) + await garden.resolveProviders(opts.force) printFooter(footerLog) diff --git a/garden-service/src/commands/plugins.ts b/garden-service/src/commands/plugins.ts index dd7a3803c2..5872fb0479 100644 --- a/garden-service/src/commands/plugins.ts +++ b/garden-service/src/commands/plugins.ts @@ -7,13 +7,14 @@ */ import chalk from "chalk" -import { max, padEnd, fromPairs } from "lodash" +import { max, padEnd, fromPairs, zip } from "lodash" import { findByName } from "../util/util" import { dedent } from "../util/string" import { ParameterError, toGardenError } from "../exceptions" import { LogEntry } from "../logger/log-entry" import { Garden } from "../garden" import { Command, CommandResult, CommandParams, StringParameter } from "./base" +import * as Bluebird from "bluebird" const pluginArgs = { plugin: new StringParameter({ @@ -62,7 +63,7 @@ export class PluginsCommand extends Command { } // We're executing a command - const plugin = garden.getPlugin(args.plugin) + const plugin = await garden.getPlugin(args.plugin) const command = findByName(plugin.commands || [], args.command) if (!command) { @@ -76,7 +77,8 @@ export class PluginsCommand extends Command { } } - const ctx = await garden.getPluginContext(args.plugin) + const provider = await garden.resolveProvider(args.plugin) + const ctx = await garden.getPluginContext(provider) try { const { result, errors = [] } = await command.handler({ ctx, log }) @@ -90,12 +92,12 @@ export class PluginsCommand extends Command { async function listPlugins(garden: Garden, log: LogEntry, pluginsToList: string[]) { log.info(chalk.white.bold("PLUGIN COMMANDS")) - for (const pluginName of pluginsToList) { - const plugin = garden.getPlugin(pluginName) + const plugins = await Bluebird.map(pluginsToList, async (pluginName) => { + const plugin = await garden.getPlugin(pluginName) const commands = plugin.commands || [] if (commands.length === 0) { - continue + return plugin } const maxNameLength = max(commands.map(c => c.name.length))! @@ -107,8 +109,10 @@ async function listPlugins(garden: Garden, log: LogEntry, pluginsToList: string[ // Line between different plugins log.info("") - } - const result = fromPairs(pluginsToList.map(name => [name, garden.getPlugin(name).commands || []])) + return plugin + }) + + const result = fromPairs(zip(pluginsToList, plugins)) return { result } } diff --git a/garden-service/src/commands/run/module.ts b/garden-service/src/commands/run/module.ts index faca2784b4..b4d9f75820 100644 --- a/garden-service/src/commands/run/module.ts +++ b/garden-service/src/commands/run/module.ts @@ -88,7 +88,6 @@ export class RunModuleCommand extends Command { printHeader(headerLog, msg, "runner") const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) await garden.processTasks([buildTask]) diff --git a/garden-service/src/commands/run/service.ts b/garden-service/src/commands/run/service.ts index 7b30d161c8..0a6df847ef 100644 --- a/garden-service/src/commands/run/service.ts +++ b/garden-service/src/commands/run/service.ts @@ -65,7 +65,6 @@ export class RunServiceCommand extends Command { ) const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) await garden.processTasks([buildTask]) diff --git a/garden-service/src/commands/run/task.ts b/garden-service/src/commands/run/task.ts index ef1485e34d..4e9cbe74cf 100644 --- a/garden-service/src/commands/run/task.ts +++ b/garden-service/src/commands/run/task.ts @@ -59,9 +59,6 @@ export class RunTaskCommand extends Command { printHeader(headerLog, msg, "runner") - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) - const taskTask = await TaskTask.factory({ garden, graph, task, log, force: true, forceBuild: opts["force-build"] }) const result = (await garden.processTasks([taskTask]))[taskTask.getKey()] diff --git a/garden-service/src/commands/run/test.ts b/garden-service/src/commands/run/test.ts index 612a91af24..5a2377944f 100644 --- a/garden-service/src/commands/run/test.ts +++ b/garden-service/src/commands/run/test.ts @@ -91,7 +91,6 @@ export class RunTestCommand extends Command { ) const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) const buildTask = new BuildTask({ garden, log, module, force: opts["force-build"] }) await garden.processTasks([buildTask]) diff --git a/garden-service/src/commands/test.ts b/garden-service/src/commands/test.ts index ce7d4a45ca..aa353a4ea5 100644 --- a/garden-service/src/commands/test.ts +++ b/garden-service/src/commands/test.ts @@ -101,9 +101,6 @@ export class TestCommand extends Command { modules = await graph.getModules() } - const actions = await garden.getActionHelper() - await actions.prepareEnvironment({ log }) - const filterNames = opts.name ? [opts.name] : [] const force = opts.force const forceBuild = opts["force-build"] diff --git a/garden-service/src/config/config-context.ts b/garden-service/src/config/config-context.ts index b52efb63f2..114301d8a2 100644 --- a/garden-service/src/config/config-context.ts +++ b/garden-service/src/config/config-context.ts @@ -249,17 +249,17 @@ class ProviderContext extends ConfigContext { ) public config: ProviderConfig - // TODO: Need further steps to be able to reference runtime outputs for providers. - // @schema( - // joiIdentifierMap(joiPrimitive()) - // .description("The outputs defined by the provider (see individual plugin docs for details).") - // .example({ "cluster-ip": "1.2.3.4" }), - // ) - // public outputs: PrimitiveMap - - constructor(root: ConfigContext, config: ProviderConfig) { + @schema( + joiIdentifierMap(joiPrimitive()) + .description("The outputs defined by the provider (see individual plugin docs for details).") + .example({ "cluster-ip": "1.2.3.4" }), + ) + public outputs: PrimitiveMap + + constructor(root: ConfigContext, provider: Provider) { super(root) - this.config = config + this.config = provider.config + this.outputs = provider.status.outputs } } diff --git a/garden-service/src/config/provider.ts b/garden-service/src/config/provider.ts index a4b89f918b..d39cc53a16 100644 --- a/garden-service/src/config/provider.ts +++ b/garden-service/src/config/provider.ts @@ -13,6 +13,8 @@ import { ConfigurationError } from "../exceptions" import { ModuleConfig, moduleConfigSchema } from "./module" import { uniq } from "lodash" import { GardenPlugin } from "../types/plugin/plugin" +import { EnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus" +import { environmentStatusSchema } from "./status" export interface ProviderConfig { name: string @@ -45,6 +47,7 @@ export interface Provider { environments?: string[] moduleConfigs: ModuleConfig[] config: T + status: EnvironmentStatus } export const providerSchema = providerFixedFieldsSchema @@ -54,6 +57,7 @@ export const providerSchema = providerFixedFieldsSchema config: joi.lazy(() => providerConfigBaseSchema) .required(), moduleConfigs: joiArray(moduleConfigSchema.optional()), + status: environmentStatusSchema, }) export const providersSchema = joiArray(providerSchema) @@ -71,16 +75,18 @@ export const defaultProvider: Provider = { dependencies: [], moduleConfigs: [], config: { name: "_default" }, + status: { ready: true, outputs: {} }, } export function providerFromConfig( - config: ProviderConfig, dependencies: Provider[], moduleConfigs: ModuleConfig[], + config: ProviderConfig, dependencies: Provider[], moduleConfigs: ModuleConfig[], status: EnvironmentStatus, ): Provider { return { name: config.name, dependencies, moduleConfigs, config, + status, } } @@ -91,12 +97,12 @@ export async function getProviderDependencies(plugin: GardenPlugin, config: Prov const references = await collectTemplateReferences(config) for (const key of references) { - if (key[0] === "provider") { + if (key[0] === "providers") { const providerName = key[1] if (!providerName) { throw new ConfigurationError(deline` Invalid template key '${key.join(".")}' in configuration for provider '${config.name}'. You must - specify a provider name as well (e.g. \${provider.my-provider}). + specify a provider name as well (e.g. \${providers.my-provider}). `, { config, key: key.join(".") }, ) } diff --git a/garden-service/src/config/dashboard.ts b/garden-service/src/config/status.ts similarity index 62% rename from garden-service/src/config/dashboard.ts rename to garden-service/src/config/status.ts index a61b9bd10e..aba4385ff3 100644 --- a/garden-service/src/config/dashboard.ts +++ b/garden-service/src/config/status.ts @@ -6,7 +6,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { joiArray, joi } from "./common" +import { joiArray, joi, joiVariables } from "./common" export interface DashboardPage { title: string @@ -36,4 +36,21 @@ export const dashboardPageSchema = joi.object() }) export const dashboardPagesSchema = joiArray(dashboardPageSchema) + .optional() .description("One or more pages to add to the Garden dashboard.") + +export const environmentStatusSchema = joi.object() + .keys({ + ready: joi.boolean() + .required() + .description("Set to true if the environment is fully configured for a provider."), + dashboardPages: dashboardPagesSchema, + detail: joi.object() + .optional() + .meta({ extendable: true }) + .description("Use this to include additional information that is specific to the provider."), + outputs: joiVariables() + .meta({ extendable: true }) + .description("Output variables that modules and other variables can reference."), + }) + .description("Description of an environment's status for a provider.") diff --git a/garden-service/src/docs/config.ts b/garden-service/src/docs/config.ts index cc9fa7f699..29ac2ecb4c 100644 --- a/garden-service/src/docs/config.ts +++ b/garden-service/src/docs/config.ts @@ -7,32 +7,21 @@ */ import Joi = require("@hapi/joi") -import { - readFileSync, - writeFileSync, -} from "fs" +import { readFileSync, writeFileSync } from "fs" import { safeDump } from "js-yaml" import * as linewrap from "linewrap" import { resolve } from "path" -import { - get, - flatten, - uniq, - startCase, -} from "lodash" +import { get, flatten, startCase, uniq } from "lodash" import { projectSchema } from "../config/project" import { baseModuleSpecSchema } from "../config/module" import handlebars = require("handlebars") -import { configSchema as localK8sConfigSchema } from "../plugins/kubernetes/local/config" -import { configSchema as k8sConfigSchema } from "../plugins/kubernetes/config" -import { configSchema as openfaasConfigSchema } from "../plugins/openfaas/config" import { joiArray, joi } from "../config/common" -import { mavenContainerConfigSchema } from "../plugins/maven-container/maven-container" import { Garden } from "../garden" import { GARDEN_SERVICE_ROOT } from "../constants" import { indent, renderMarkdownTable } from "./util" import { ModuleContext } from "../config/config-context" import { defaultDotIgnoreFiles } from "../util/fs" +import { providerConfigBaseSchema } from "../config/provider" export const TEMPLATES_DIR = resolve(GARDEN_SERVICE_ROOT, "src", "docs", "templates") @@ -54,14 +43,6 @@ const moduleTypes = [ { name: "openfaas", pluginName: "local-kubernetes" }, ] -const providers = [ - { name: "local-kubernetes", schema: localK8sConfigSchema }, - { name: "kubernetes", schema: k8sConfigSchema }, - { name: "local-openfaas", schema: openfaasConfigSchema }, - { name: "maven-container", schema: mavenContainerConfigSchema }, - { name: "openfaas", schema: openfaasConfigSchema }, -] - interface RenderOpts { level?: number showRequired?: boolean @@ -404,11 +385,15 @@ export function renderConfigReference(configSchema: Joi.ObjectSchema, titlePrefi * Generates the provider reference from the provider.hbs template. * The reference includes the rendered output from the config-partial.hbs template. */ -function renderProviderReference(schema: Joi.ObjectSchema, name: string) { +function renderProviderReference(schema: Joi.ObjectSchema, name: string, outputsSchema?: Joi.ObjectSchema) { const providerTemplatePath = resolve(TEMPLATES_DIR, "provider.hbs") const { markdownReference, yaml } = renderConfigReference(schema) + + const outputsReference = outputsSchema + && renderConfigReference(outputsSchema, "providers..").markdownReference + const template = handlebars.compile(readFileSync(providerTemplatePath).toString()) - return template({ name, markdownReference, yaml }) + return template({ name, markdownReference, yaml, outputsReference }) } /** @@ -443,7 +428,6 @@ export async function writeConfigReferenceDocs(docsRoot: string) { const referenceDir = resolve(docsRoot, "reference") const configPath = resolve(referenceDir, "config.md") - const moduleProviders = uniq(moduleTypes.map(m => m.pluginName || m.name)).map(name => ({ name })) const garden = await Garden.factory(__dirname, { config: { path: __dirname, @@ -452,24 +436,35 @@ export async function writeConfigReferenceDocs(docsRoot: string) { name: "generate-docs", defaultEnvironment: "default", dotIgnoreFiles: defaultDotIgnoreFiles, - providers: moduleProviders, variables: {}, environments: [ { name: "default", - providers: [], variables: {}, }, ], + providers: [ + { name: "local-kubernetes" }, + { name: "kubernetes" }, + { name: "local-openfaas" }, + { name: "maven-container" }, + { name: "openfaas" }, + ], }, }) - // Render provider docs const providerDir = resolve(referenceDir, "providers") - for (const { name, schema } of providers) { + for (const [name, plugin] of Object.entries(await garden.getPlugins())) { + // Currently nothing to document for these + if (name === "container" || name === "exec") { + continue + } + const path = resolve(providerDir, `${name}.md`) console.log("->", path) - writeFileSync(path, renderProviderReference(populateProviderSchema(schema), name)) + const schema = populateProviderSchema(plugin.configSchema || providerConfigBaseSchema) + const outputsSchema = plugin.outputsSchema + writeFileSync(path, renderProviderReference(schema, name, outputsSchema)) } // Render module type docs diff --git a/garden-service/src/docs/templates/provider.hbs b/garden-service/src/docs/templates/provider.hbs index 4ee2503667..16de037750 100644 --- a/garden-service/src/docs/templates/provider.hbs +++ b/garden-service/src/docs/templates/provider.hbs @@ -14,3 +14,10 @@ The values in the schema below are the default values. ```yaml {{{yaml}}} ``` +{{#if outputsReference}} + +## Outputs + +The following keys are available via the `${providers.}` template string key for `{{{name}}}` +providers. +{{{outputsReference}}}{{/if}} \ No newline at end of file diff --git a/garden-service/src/garden.ts b/garden-service/src/garden.ts index 42be980508..ea5a1d3f15 100644 --- a/garden-service/src/garden.ts +++ b/garden-service/src/garden.ts @@ -8,7 +8,7 @@ import Bluebird = require("bluebird") import { parse, relative, resolve, sep, dirname } from "path" -import { flatten, isString, cloneDeep, sortBy, set, zip } from "lodash" +import { flatten, isString, cloneDeep, sortBy, set, fromPairs, keyBy } from "lodash" const AsyncLock = require("async-lock") import { TreeCache } from "./cache" @@ -22,7 +22,7 @@ import { VcsHandler, ModuleVersion } from "./vcs/vcs" import { GitHandler } from "./vcs/git" import { BuildDir } from "./build-dir" import { ConfigGraph } from "./config-graph" -import { TaskGraph, TaskResults } from "./task-graph" +import { TaskGraph, TaskResults, ProcessTasksOpts } from "./task-graph" import { getLogger } from "./logger/logger" import { PluginActions, PluginFactory, GardenPlugin } from "./types/plugin/plugin" import { joiIdentifier, validate, PrimitiveMap, validateWithPath } from "./config/common" @@ -41,10 +41,11 @@ import { LogEntry } from "./logger/log-entry" import { EventBus } from "./events" import { Watcher } from "./watch" import { findConfigPathsInPath, getConfigFilePath, getWorkingCopyId } from "./util/fs" -import { Provider, ProviderConfig, getProviderDependencies } from "./config/provider" +import { Provider, ProviderConfig, getProviderDependencies, defaultProvider } from "./config/provider" import { ResolveProviderTask } from "./tasks/resolve-provider" import { ActionHelper } from "./actions" import { DependencyGraph, detectCycles, cyclesToString } from "./util/validate-dependencies" +import chalk from "chalk" export interface ActionHandlerMap { [actionName: string]: PluginActions[T] @@ -80,8 +81,6 @@ interface ModuleConfigResolveOpts extends ContextResolveOpts { configContext?: ModuleConfigContext } -const asyncLock = new AsyncLock() - export interface GardenParams { buildDir: BuildDir, environmentName: string, @@ -101,7 +100,7 @@ export interface GardenParams { export class Garden { public readonly log: LogEntry - private readonly loadedPlugins: { [key: string]: GardenPlugin } + private loadedPlugins: { [key: string]: GardenPlugin } private moduleConfigs: ModuleConfigMap private pluginModuleConfigs: ModuleConfig[] private resolvedProviders: Provider[] @@ -109,6 +108,7 @@ export class Garden { private readonly registeredPlugins: { [key: string]: PluginFactory } private readonly taskGraph: TaskGraph private watcher: Watcher + private asyncLock: any public readonly configStore: ConfigStore public readonly globalConfigStore: GlobalConfigStore @@ -145,6 +145,7 @@ export class Garden { this.dotIgnoreFiles = params.dotIgnoreFiles this.moduleIncludePatterns = params.moduleIncludePatterns this.moduleExcludePatterns = params.moduleExcludePatterns || [] + this.asyncLock = new AsyncLock() // make sure we're on a supported platform const currentPlatform = platform() @@ -167,7 +168,6 @@ export class Garden { this.cache = new TreeCache() this.moduleConfigs = {} - this.loadedPlugins = {} this.pluginModuleConfigs = [] this.registeredPlugins = {} @@ -243,16 +243,16 @@ export class Garden { this.watcher && this.watcher.stop() } - getPluginContext(providerName: string) { - return createPluginContext(this, providerName) + getPluginContext(provider: Provider) { + return createPluginContext(this, provider) } async clearBuilds() { return this.buildDir.clear() } - async processTasks(tasks: BaseTask[]): Promise { - return this.taskGraph.process(tasks) + async processTasks(tasks: BaseTask[], opts?: ProcessTasksOpts): Promise { + return this.taskGraph.process(tasks, opts) } /** @@ -327,10 +327,6 @@ export class Garden { } private async loadPlugin(pluginName: string) { - if (this.loadedPlugins[pluginName]) { - return this.loadedPlugins[pluginName] - } - this.log.silly(`Loading plugin ${pluginName}`) const factory = this.registeredPlugins[pluginName] @@ -357,15 +353,14 @@ export class Garden { plugin = validate(plugin, pluginSchema, { context: `plugin "${pluginName}"` }) - this.loadedPlugins[pluginName] = plugin - this.log.silly(`Done loading plugin ${pluginName}`) return plugin } - getPlugin(pluginName: string) { - const plugin = this.loadedPlugins[pluginName] + async getPlugin(pluginName: string) { + const plugins = await this.getPlugins() + const plugin = plugins[pluginName] if (!plugin) { throw new PluginError(`Could not find plugin '${pluginName}'. Are you missing a provider configuration?`, { @@ -377,23 +372,64 @@ export class Garden { return plugin } + async getPlugins() { + await this.asyncLock.acquire("load-plugins", async () => { + if (this.loadedPlugins) { + return + } + + this.log.silly(`Loading plugins`) + const rawConfigs = this.getRawProviderConfigs() + const plugins = {} + + await Bluebird.map(rawConfigs, async (config) => { + plugins[config.name] = await this.loadPlugin(config.name) + }) + + this.loadedPlugins = plugins + this.log.silly(`Loaded plugins: ${Object.keys(plugins).join(", ")}`) + }) + + return this.loadedPlugins + } + getRawProviderConfigs() { return this.providerConfigs } - async resolveProviders(): Promise { - await asyncLock.acquire("resolve-providers", async () => { + async resolveProvider(name: string) { + if (name === "_default") { + return defaultProvider + } + + const providers = await this.resolveProviders() + const provider = findByName(providers, name) + + if (!provider) { + throw new PluginError(`Could not find provider '${name}'`, { name, providers }) + } + + return provider + } + + async resolveProviders(forceInit = false): Promise { + await this.asyncLock.acquire("resolve-providers", async () => { if (this.resolvedProviders) { return } + this.log.silly(`Resolving providers`) + const log = this.log.info({ section: "providers", msg: "Getting status...", status: "active" }) + const rawConfigs = this.getRawProviderConfigs() - const plugins = await Bluebird.map(rawConfigs, async (config) => this.loadPlugin(config.name)) + const configsByName = keyBy(rawConfigs, "name") + const plugins = Object.entries(await this.getPlugins()) // Detect circular deps here const pluginGraph: DependencyGraph = {} - await Bluebird.map(zip(plugins, rawConfigs), async ([plugin, config]) => { + await Bluebird.map(plugins, async ([name, plugin]) => { + const config = configsByName[name] for (const dep of await getProviderDependencies(plugin!, config!)) { set(pluginGraph, [config!.name, dep], { distance: 1, next: dep }) } @@ -410,7 +446,7 @@ export class Garden { ) } - const tasks = rawConfigs.map((config, i) => { + const tasks = plugins.map(([name, plugin]) => { // TODO: actually resolve version, based on the VCS version of the plugin and its dependencies const version = { versionString: getPackageVersion(), @@ -420,17 +456,20 @@ export class Garden { files: [], } - const plugin = plugins[i] + const config = configsByName[name] return new ResolveProviderTask({ garden: this, - log: this.log, + log, plugin, config, version, + forceInit, }) }) - const taskResults = await this.processTasks(tasks) + + // Process as many providers in parallel as possible + const taskResults = await this.processTasks(tasks, { concurrencyLimit: plugins.length }) const failed = Object.values(taskResults).filter(r => r && r.error) @@ -442,24 +481,37 @@ export class Garden { ) } - this.resolvedProviders = Object.values(taskResults).map(result => result.output) + const providers: Provider[] = Object.values(taskResults).map(result => result.output) - await Bluebird.map(this.resolvedProviders, async (provider) => + await Bluebird.map(providers, async (provider) => Bluebird.map(provider.moduleConfigs, async (moduleConfig) => { // Make sure module and all nested entities are scoped to the plugin moduleConfig.plugin = provider.name return this.addModule(moduleConfig) }), ) + + this.resolvedProviders = providers + + log.setSuccess({ msg: chalk.green("Done"), append: true }) + this.log.silly(`Resolved providers: ${providers.map(p => p.name).join(", ")}`) }) return this.resolvedProviders } + /** + * Returns the reported status from all configured providers. + */ + async getEnvironmentStatus() { + const providers = await this.resolveProviders() + return fromPairs(providers.map(p => [p.name, p.status])) + } + async getActionHelper() { if (!this.actionHelper) { - const providers = await this.resolveProviders() - this.actionHelper = new ActionHelper(this, providers) + const plugins = await this.getPlugins() + this.actionHelper = new ActionHelper(this, plugins) } return this.actionHelper @@ -486,13 +538,14 @@ export class Garden { */ async resolveModuleConfigs(keys?: string[], opts: ModuleConfigResolveOpts = {}): Promise { const actions = await this.getActionHelper() + const providers = await this.resolveProviders() const configs = await this.getRawModuleConfigs(keys) if (!opts.configContext) { opts.configContext = new ModuleConfigContext( this, this.environmentName, - await this.resolveProviders(), + providers, this.variables, Object.values(this.moduleConfigs), ) @@ -546,7 +599,8 @@ export class Garden { moduleType: config.type, }) - const ctx = await this.getPluginContext(configureHandler["pluginName"]) + const provider = await this.resolveProvider(configureHandler["pluginName"]) + const ctx = await this.getPluginContext(provider) config = await configureHandler({ ctx, moduleConfig: config, log: this.log }) if (config.plugin) { @@ -642,7 +696,7 @@ export class Garden { Scans the project root for modules and adds them to the context. */ async scanModules(force = false) { - return asyncLock.acquire("scan-modules", async () => { + return this.asyncLock.acquire("scan-modules", async () => { if (this.modulesScanned && !force) { return } diff --git a/garden-service/src/plugin-context.ts b/garden-service/src/plugin-context.ts index 7b5361b179..5c88653a7a 100644 --- a/garden-service/src/plugin-context.ts +++ b/garden-service/src/plugin-context.ts @@ -7,10 +7,9 @@ */ import { Garden } from "./garden" -import { keyBy, cloneDeep } from "lodash" +import { cloneDeep } from "lodash" import { projectNameSchema, projectSourcesSchema, environmentNameSchema } from "./config/project" -import { PluginError } from "./exceptions" -import { defaultProvider, Provider, providerSchema, ProviderConfig } from "./config/provider" +import { Provider, providerSchema, ProviderConfig } from "./config/provider" import { configStoreSchema } from "./config-store" import { deline } from "./util/string" import { joi } from "./config/common" @@ -52,18 +51,7 @@ export const pluginContextSchema = joi.object() .description("A unique ID assigned to the current project working copy."), }) -export async function createPluginContext(garden: Garden, providerName: string): Promise { - const providers = keyBy(await garden.resolveProviders(), "name") - let provider = providers[providerName] - - if (providerName === "_default") { - provider = defaultProvider - } - - if (!provider) { - throw new PluginError(`Could not find provider '${providerName}'`, { providerName, providers }) - } - +export function createPluginContext(garden: Garden, provider: Provider): PluginContext { return { environmentName: garden.environmentName, projectName: garden.projectName, diff --git a/garden-service/src/plugins/google/common.ts b/garden-service/src/plugins/google/common.ts index 4b37b7401f..fb1bba3030 100644 --- a/garden-service/src/plugins/google/common.ts +++ b/garden-service/src/plugins/google/common.ts @@ -7,12 +7,13 @@ */ import { Module } from "../../types/module" -import { PrepareEnvironmentParams } from "../../types/plugin/provider/prepareEnvironment" +import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "../../types/plugin/provider/prepareEnvironment" import { ConfigurationError } from "../../exceptions" import { ExecTestSpec } from "../exec" import { GCloud } from "./gcloud" import { ModuleSpec } from "../../config/module" import { CommonServiceSpec } from "../../config/service" +import { EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" export const GOOGLE_CLOUD_DEFAULT_REGION = "us-central1" @@ -23,9 +24,9 @@ export interface GoogleCloudModule< > extends Module { } export async function getEnvironmentStatus() { - let sdkInfo + let sdkInfo: any - const output = { + const output: EnvironmentStatus = { ready: true, detail: { sdkInstalled: true, @@ -33,6 +34,7 @@ export async function getEnvironmentStatus() { betaComponentsInstalled: true, sdkInfo: {}, }, + outputs: {}, } try { @@ -55,7 +57,7 @@ export async function getEnvironmentStatus() { return output } -export async function prepareEnvironment({ status, log }: PrepareEnvironmentParams) { +export async function prepareEnvironment({ status, log }: PrepareEnvironmentParams): Promise { if (!status.detail.sdkInstalled) { throw new ConfigurationError( "Google Cloud SDK is not installed. " + @@ -81,7 +83,7 @@ export async function prepareEnvironment({ status, log }: PrepareEnvironmentPara await gcloud().call(["init"], { timeout: 600, tty: true }) } - return {} + return { status: { ready: true, outputs: {} } } } export function gcloud(project?: string, account?: string) { diff --git a/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts b/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts index 1c263b679a..cdb9dd928c 100644 --- a/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts +++ b/garden-service/src/plugins/kubernetes/commands/uninstall-garden-services.ts @@ -26,7 +26,7 @@ export const uninstallGardenServices: PluginCommand = { const k8sCtx = ctx const variables = getKubernetesSystemVariables(k8sCtx.provider.config) - const sysGarden = await getSystemGarden(k8sCtx, variables || {}) + const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) const actions = await sysGarden.getActionHelper() const result = await actions.deleteEnvironment(entry) diff --git a/garden-service/src/plugins/kubernetes/helm/build.ts b/garden-service/src/plugins/kubernetes/helm/build.ts index e84650e493..2fe9133910 100644 --- a/garden-service/src/plugins/kubernetes/helm/build.ts +++ b/garden-service/src/plugins/kubernetes/helm/build.ts @@ -20,7 +20,6 @@ import { BuildModuleParams, BuildResult } from "../../../types/plugin/module/bui export async function buildHelmModule({ ctx, module, log }: BuildModuleParams): Promise { const k8sCtx = ctx const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, diff --git a/garden-service/src/plugins/kubernetes/helm/common.ts b/garden-service/src/plugins/kubernetes/helm/common.ts index 73cadd381e..c3cc19a8fe 100644 --- a/garden-service/src/plugins/kubernetes/helm/common.ts +++ b/garden-service/src/plugins/kubernetes/helm/common.ts @@ -45,7 +45,6 @@ export async function getChartResources(ctx: PluginContext, module: Module, log: const valuesPath = getValuesPath(chartPath) const k8sCtx = ctx const namespace = await getNamespace({ - configStore: k8sCtx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, @@ -280,7 +279,6 @@ async function renderHelmTemplateString( const valuesPath = getValuesPath(chartPath) const k8sCtx = ctx const namespace = await getNamespace({ - configStore: k8sCtx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, diff --git a/garden-service/src/plugins/kubernetes/init.ts b/garden-service/src/plugins/kubernetes/init.ts index d3a5dab370..6b9480949f 100644 --- a/garden-service/src/plugins/kubernetes/init.ts +++ b/garden-service/src/plugins/kubernetes/init.ts @@ -18,7 +18,7 @@ import { systemNamespace, } from "./system" import { GetEnvironmentStatusParams, EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" -import { PrepareEnvironmentParams } from "../../types/plugin/provider/prepareEnvironment" +import { PrepareEnvironmentParams, PrepareEnvironmentResult } from "../../types/plugin/provider/prepareEnvironment" import { CleanupEnvironmentParams } from "../../types/plugin/provider/cleanupEnvironment" import { millicpuToString, megabytesToString } from "./util" import chalk from "chalk" @@ -35,17 +35,10 @@ import { combineStates, ServiceStatusMap } from "../../types/service" */ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusParams): Promise { const k8sCtx = ctx - const variables = getKubernetesSystemVariables(k8sCtx.provider.config) - - const sysGarden = await getSystemGarden(k8sCtx, variables || {}) - const sysCtx = await sysGarden.getPluginContext(k8sCtx.provider.name) let projectReady = true - // Ensure project and system namespaces. We need the system namespace independent of system services - // because we store test results in the system metadata namespace. - await prepareNamespaces({ ctx, log }) - await prepareNamespaces({ ctx: sysCtx, log }) + const namespaces = await prepareNamespaces({ ctx, log }) // Check Tiller status in project namespace if (await checkTillerStatus(k8sCtx, k8sCtx.provider, log) !== "ready") { @@ -66,13 +59,26 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar ready: projectReady, detail, dashboardPages: [], + outputs: { + ...namespaces, + }, } - // No need to continue if we don't need any system services - if (systemServiceNames.length === 0) { + if ( + // No need to continue if we don't need any system services + systemServiceNames.length === 0 + || + // Make sure we don't recurse infinitely + k8sCtx.provider.config.namespace === systemNamespace + ) { return result } + const variables = getKubernetesSystemVariables(k8sCtx.provider.config) + const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) + const sysProvider = await sysGarden.resolveProvider(k8sCtx.provider.name) + const sysCtx = await sysGarden.getPluginContext(sysProvider) + // Check Tiller status in system namespace const tillerStatus = await checkTillerStatus(sysCtx, sysCtx.provider, log) @@ -104,6 +110,8 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar detail.serviceStatuses = systemServiceStatus.serviceStatuses detail.systemServiceState = systemServiceStatus.state + sysGarden.log.setSuccess() + return result } @@ -113,7 +121,7 @@ export async function getEnvironmentStatus({ ctx, log }: GetEnvironmentStatusPar * 2. Installs Tiller in system namespace (if provider has system services) * 3. Deploys system services (if provider has system services) */ -export async function prepareEnvironment(params: PrepareEnvironmentParams) { +export async function prepareEnvironment(params: PrepareEnvironmentParams): Promise { const { ctx, log, force } = params const k8sCtx = ctx @@ -123,7 +131,7 @@ export async function prepareEnvironment(params: PrepareEnvironmentParams) { // Prepare system services await prepareSystem({ ...params, clusterInit: false }) - return {} + return { status: { ready: true, outputs: {} } } } export async function prepareSystem( @@ -174,8 +182,10 @@ export async function prepareSystem( } // Install Tiller to system namespace - const sysGarden = await getSystemGarden(k8sCtx, variables || {}) - const sysCtx = await sysGarden.getPluginContext(k8sCtx.provider.name) + const sysGarden = await getSystemGarden(k8sCtx, variables || {}, log) + const sysProvider = await sysGarden.resolveProvider(k8sCtx.provider.name) + const sysCtx = await sysGarden.getPluginContext(sysProvider) + await installTiller({ ctx: sysCtx, provider: sysCtx.provider, log, force }) // Install system services @@ -188,6 +198,8 @@ export async function prepareSystem( serviceNames: systemServiceNames, }) + sysGarden.log.setSuccess() + return {} } diff --git a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts index 5b3161d1fe..6798e6cc1d 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes-module/handlers.ts @@ -50,7 +50,6 @@ async function getServiceStatus( ): Promise { const k8sCtx = ctx const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, @@ -76,7 +75,6 @@ async function deployService( const k8sCtx = ctx const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: k8sCtx.projectName, provider: k8sCtx.provider, diff --git a/garden-service/src/plugins/kubernetes/kubernetes.ts b/garden-service/src/plugins/kubernetes/kubernetes.ts index 98c0459d65..c7b60a4add 100644 --- a/garden-service/src/plugins/kubernetes/kubernetes.ts +++ b/garden-service/src/plugins/kubernetes/kubernetes.ts @@ -26,6 +26,7 @@ import { cleanupClusterRegistry } from "./commands/cleanup-cluster-registry" import { clusterInit } from "./commands/cluster-init" import { uninstallGardenServices } from "./commands/uninstall-garden-services" import chalk from "chalk" +import { joi, joiIdentifier } from "../../config/common" export const name = "kubernetes" @@ -94,9 +95,20 @@ export async function debugInfo({ ctx, log, includeProject }: GetDebugInfoParams } } +const outputsSchema = joi.object() + .keys({ + "app-namespace": joiIdentifier() + .required() + .description("The primary namespace used for resource deployments."), + "metadata-namespace": joiIdentifier() + .required() + .description("The namespace used for Garden metadata."), + }) + export function gardenPlugin(): GardenPlugin { return { configSchema, + outputsSchema, commands: [ cleanupClusterRegistry, clusterInit, diff --git a/garden-service/src/plugins/kubernetes/namespace.ts b/garden-service/src/plugins/kubernetes/namespace.ts index cd0ed561a2..bf0e91fa23 100644 --- a/garden-service/src/plugins/kubernetes/namespace.ts +++ b/garden-service/src/plugins/kubernetes/namespace.ts @@ -12,13 +12,11 @@ import { intersection } from "lodash" import { PluginContext } from "../../plugin-context" import { KubeApi } from "./api" import { KubernetesProvider, KubernetesPluginContext } from "./config" -import { name as providerName } from "./kubernetes" -import { AuthenticationError, DeploymentError, TimeoutError } from "../../exceptions" +import { DeploymentError, TimeoutError } from "../../exceptions" import { getPackageVersion, sleep } from "../../util/util" import { GetEnvironmentStatusParams } from "../../types/plugin/provider/getEnvironmentStatus" import { kubectl, KUBECTL_DEFAULT_TIMEOUT } from "./kubectl" import { LogEntry } from "../../logger/log-entry" -import { ConfigStore } from "../../config-store" import { gardenAnnotationKey } from "../../util/string" const GARDEN_VERSION = getPackageVersion() @@ -61,7 +59,6 @@ export async function createNamespace(api: KubeApi, namespace: string) { } interface GetNamespaceParams { - configStore: ConfigStore, log: LogEntry, projectName: string, provider: KubernetesProvider, @@ -70,33 +67,9 @@ interface GetNamespaceParams { } export async function getNamespace( - { projectName, configStore: localConfigStore, log, provider, suffix, skipCreate }: GetNamespaceParams, + { projectName, log, provider, suffix, skipCreate }: GetNamespaceParams, ): Promise { - let namespace - - if (provider.config.namespace !== undefined) { - namespace = provider.config.namespace - } else { - // Note: The local-kubernetes always defines a namespace name, so this logic only applies to the kubernetes provider - // TODO: Move this logic out to the kubernetes plugin init/validation - const localConfig = await localConfigStore.get() - const k8sConfig = localConfig.kubernetes || {} - let { username, ["previous-usernames"]: previousUsernames } = k8sConfig - - if (!username) { - username = provider.config.defaultUsername - } - - if (!username) { - throw new AuthenticationError( - `User not logged into provider ${providerName}. Please specify defaultUsername in provider ` + - `config or run garden init.`, - { previousUsernames, provider: providerName }, - ) - } - - namespace = `${username}--${projectName}` - } + let namespace = provider.config.namespace || projectName if (suffix) { namespace = `${namespace}--${suffix}` @@ -112,7 +85,6 @@ export async function getNamespace( export async function getAppNamespace(ctx: PluginContext, log: LogEntry, provider: KubernetesProvider) { return getNamespace({ - configStore: ctx.configStore, log, projectName: ctx.projectName, provider, @@ -121,7 +93,6 @@ export async function getAppNamespace(ctx: PluginContext, log: LogEntry, provide export function getMetadataNamespace(ctx: PluginContext, log: LogEntry, provider: KubernetesProvider) { return getNamespace({ - configStore: ctx.configStore, log, projectName: ctx.projectName, provider, @@ -163,10 +134,10 @@ export async function prepareNamespaces({ ctx, log }: GetEnvironmentStatusParams ) } - await Bluebird.all([ - getMetadataNamespace(k8sCtx, log, k8sCtx.provider), - getAppNamespace(k8sCtx, log, k8sCtx.provider), - ]) + return Bluebird.props({ + "app-namespace": getAppNamespace(k8sCtx, log, k8sCtx.provider), + "metadata-namespace": getMetadataNamespace(k8sCtx, log, k8sCtx.provider), + }) } export async function deleteNamespaces(namespaces: string[], api: KubeApi, log?: LogEntry) { diff --git a/garden-service/src/plugins/kubernetes/system.ts b/garden-service/src/plugins/kubernetes/system.ts index 4c452677c9..bb592fda1a 100644 --- a/garden-service/src/plugins/kubernetes/system.ts +++ b/garden-service/src/plugins/kubernetes/system.ts @@ -21,7 +21,7 @@ import { getPackageVersion } from "../../util/util" import { deline, gardenAnnotationKey } from "../../util/string" import { deleteNamespaces } from "./namespace" import { PluginError } from "../../exceptions" -import { DashboardPage } from "../../config/dashboard" +import { DashboardPage } from "../../config/status" import { PrimitiveMap } from "../../config/common" import { combineStates } from "../../types/service" import { KubernetesResource } from "./types" @@ -41,7 +41,9 @@ export const systemMetadataNamespace = "garden-system--metadata" * stored at the project level. This way we can run several Garden processes at the same time * without them all modifying the same system build directory, which can cause unexpected issues. */ -export async function getSystemGarden(ctx: KubernetesPluginContext, variables: PrimitiveMap): Promise { +export async function getSystemGarden( + ctx: KubernetesPluginContext, variables: PrimitiveMap, log: LogEntry, +): Promise { const sysProvider: KubernetesConfig = { ...ctx.provider.config, environments: ["default"], @@ -66,6 +68,7 @@ export async function getSystemGarden(ctx: KubernetesPluginContext, variables: P providers: [sysProvider], variables, }, + log: log.info({ section: "garden-system", msg: "Initializing...", status: "active", indent: 1 }), }) } diff --git a/garden-service/src/plugins/local/local-docker-swarm.ts b/garden-service/src/plugins/local/local-docker-swarm.ts index 6cd995df0b..35e0bf5ac5 100644 --- a/garden-service/src/plugins/local/local-docker-swarm.ts +++ b/garden-service/src/plugins/local/local-docker-swarm.ts @@ -19,6 +19,7 @@ import { containerHelpers } from "../container/helpers" import { DeployServiceParams } from "../../types/plugin/service/deployService" import { ExecInServiceParams } from "../../types/plugin/service/execInService" import { GetServiceStatusParams } from "../../types/plugin/service/getServiceStatus" +import { EnvironmentStatus } from "../../types/plugin/provider/getEnvironmentStatus" // should this be configurable and/or global across providers? const DEPLOY_TIMEOUT = 30 @@ -212,7 +213,7 @@ export const gardenPlugin = (): GardenPlugin => ({ }, }) -async function getEnvironmentStatus() { +async function getEnvironmentStatus(): Promise { const docker = getDocker() try { @@ -220,13 +221,14 @@ async function getEnvironmentStatus() { return { ready: true, + outputs: {}, } } catch (err) { if (err.statusCode === 503) { // swarm has not been initialized return { ready: false, - services: [], + outputs: {}, } } else { throw err @@ -236,7 +238,7 @@ async function getEnvironmentStatus() { async function prepareEnvironment() { await getDocker().swarmInit({}) - return {} + return { status: { ready: true, dashboardPages: [], outputs: {} } } } async function getServiceStatus({ ctx, service }: GetServiceStatusParams): Promise { diff --git a/garden-service/src/plugins/openfaas/config.ts b/garden-service/src/plugins/openfaas/config.ts index ca920504f2..c70b07f2dc 100644 --- a/garden-service/src/plugins/openfaas/config.ts +++ b/garden-service/src/plugins/openfaas/config.ts @@ -11,10 +11,10 @@ import { join } from "path" import { resolve as urlResolve } from "url" import { ConfigurationError } from "../../exceptions" import { PluginContext } from "../../plugin-context" -import { joiArray, joiProviderName, joi } from "../../config/common" +import { joiArray, joiProviderName, joi, joiEnvVars } from "../../config/common" import { Module } from "../../types/module" import { Service } from "../../types/service" -import { ExecModuleSpec, execModuleSpecSchema, ExecTestSpec } from "../exec" +import { ExecModuleSpec, ExecTestSpec, execTestSchema } from "../exec" import { KubernetesProvider } from "../kubernetes/config" import { CommonServiceSpec } from "../../config/service" import { Provider, providerConfigBaseSchema, ProviderConfig } from "../../config/provider" @@ -30,10 +30,11 @@ export interface OpenFaasModuleSpec extends ExecModuleSpec { lang: string } -export const openfaasModuleSpecSchema = execModuleSpecSchema +export const openfaasModuleSpecSchema = joi.object() .keys({ dependencies: joiArray(joi.string()) .description("The names of services/functions that this function depends on at runtime."), + env: joiEnvVars(), handler: joi.string() .default(".") .posixPath({ subPathOnly: true }) @@ -43,6 +44,8 @@ export const openfaasModuleSpecSchema = execModuleSpecSchema lang: joi.string() .required() .description("The OpenFaaS language template to use to build this function."), + tests: joiArray(execTestSchema) + .description("A list of tests to run in the module."), }) .unknown(false) .description("The module specification for an OpenFaaS module.") @@ -84,8 +87,8 @@ export type OpenFaasPluginContext = PluginContext export async function describeType() { return { docs: dedent` - Deploy [OpenFaaS](https://www.openfaas.com/) functions using Garden. Requires either the \`kubernetes\` or - \`local-kubernetes\` provider to be configured. Everything else is installed automatically. + Deploy [OpenFaaS](https://www.openfaas.com/) functions using Garden. Requires either the \`openfaas\` or + \`local-openfaas\` provider to be configured. `, outputsSchema: openfaasModuleOutputsSchema, schema: openfaasModuleSpecSchema, @@ -175,7 +178,6 @@ export async function configureModule( async function getInternalGatewayUrl(ctx: PluginContext, log: LogEntry) { const k8sProvider = getK8sProvider(ctx.provider.dependencies) const namespace = await getNamespace({ - configStore: ctx.configStore, log, projectName: ctx.projectName, provider: k8sProvider, diff --git a/garden-service/src/plugins/openfaas/openfaas.ts b/garden-service/src/plugins/openfaas/openfaas.ts index c077856e2c..3f2997f2f5 100644 --- a/garden-service/src/plugins/openfaas/openfaas.ts +++ b/garden-service/src/plugins/openfaas/openfaas.ts @@ -94,7 +94,7 @@ const templateModuleConfig: ExecModuleConfig = { } async function configureProvider( - { log, config, projectName, dependencies, configStore }: ConfigureProviderParams, + { log, config, projectName, dependencies }: ConfigureProviderParams, ): Promise { const k8sProvider = getK8sProvider(dependencies) @@ -110,7 +110,6 @@ async function configureProvider( } const namespace = await getNamespace({ - configStore, log, provider: k8sProvider, projectName, diff --git a/garden-service/src/task-graph.ts b/garden-service/src/task-graph.ts index 6865e0ae3f..471e083e8e 100644 --- a/garden-service/src/task-graph.ts +++ b/garden-service/src/task-graph.ts @@ -39,11 +39,14 @@ export interface TaskResults { [key: string]: TaskResult } -export const DEFAULT_CONCURRENCY = 6 - +const DEFAULT_CONCURRENCY = 6 const concurrencyFromEnv = process.env.GARDEN_TASK_CONCURRENCY_LIMIT -export const TASK_CONCURRENCY = (concurrencyFromEnv && parseInt(concurrencyFromEnv, 10)) || DEFAULT_CONCURRENCY +export const defaultTaskConcurrency = (concurrencyFromEnv && parseInt(concurrencyFromEnv, 10)) || DEFAULT_CONCURRENCY + +export interface ProcessTasksOpts { + concurrencyLimit?: number +} export class TaskGraph { private roots: TaskNodeMap @@ -69,7 +72,7 @@ export class TaskGraph { private resultCache: ResultCache private opQueue: PQueue - constructor(private garden: Garden, private log: LogEntry, private concurrency: number = TASK_CONCURRENCY) { + constructor(private garden: Garden, private log: LogEntry) { this.roots = new TaskNodeMap() this.index = new TaskNodeMap() this.inProgress = new TaskNodeMap() @@ -81,7 +84,7 @@ export class TaskGraph { this.logEntryMap = {} } - async process(tasks: BaseTask[]): Promise { + async process(tasks: BaseTask[], opts?: ProcessTasksOpts): Promise { for (const t of tasks) { this.latestTasks[t.getKey()] = t } @@ -97,7 +100,7 @@ export class TaskGraph { // to return the latest result for each requested task. const resultKeys = tasks.map(t => t.getKey()) - return this.opQueue.add(() => this.processTasksInternal(tasksToProcess, resultKeys)) + return this.opQueue.add(() => this.processTasksInternal(tasksToProcess, resultKeys, opts)) } /** @@ -163,7 +166,11 @@ export class TaskGraph { /** * Process the graph until it's complete. */ - private async processTasksInternal(tasks: BaseTask[], resultKeys: string[]): Promise { + private async processTasksInternal( + tasks: BaseTask[], resultKeys: string[], opts?: ProcessTasksOpts, + ): Promise { + const { concurrencyLimit = defaultTaskConcurrency } = opts || {} + for (const task of tasks) { await this.addTask(this.latestTasks[task.getKey()]) } @@ -188,7 +195,7 @@ export class TaskGraph { const batch = _this.roots.getNodes() .filter(n => !this.inProgress.contains(n)) - .slice(0, _this.concurrency - this.inProgress.length) + .slice(0, concurrencyLimit - this.inProgress.length) batch.forEach(n => this.inProgress.addNode(n)) this.rebuild() diff --git a/garden-service/src/tasks/resolve-provider.ts b/garden-service/src/tasks/resolve-provider.ts index e574897748..4e0f4e52be 100644 --- a/garden-service/src/tasks/resolve-provider.ts +++ b/garden-service/src/tasks/resolve-provider.ts @@ -6,20 +6,25 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +import chalk from "chalk" import { BaseTask, TaskParams, TaskType } from "./base" import { ProviderConfig, Provider, getProviderDependencies, providerFromConfig } from "../config/provider" import { resolveTemplateStrings } from "../template-string" -import { ConfigurationError } from "../exceptions" +import { ConfigurationError, PluginError } from "../exceptions" import { keyBy } from "lodash" import { TaskResults } from "../task-graph" import { ProviderConfigContext } from "../config/config-context" import { ModuleConfig } from "../config/module" import { GardenPlugin } from "../types/plugin/plugin" import { validateWithPath } from "../config/common" +import * as Bluebird from "bluebird" +import { createPluginContext } from "../plugin-context" +import { defaultEnvironmentStatus } from "../types/plugin/provider/getEnvironmentStatus" interface Params extends TaskParams { plugin: GardenPlugin config: ProviderConfig + forceInit: boolean } /** @@ -30,11 +35,13 @@ export class ResolveProviderTask extends BaseTask { private config: ProviderConfig private plugin: GardenPlugin + private forceInit: boolean constructor(params: Params) { super(params) this.config = params.config this.plugin = params.plugin + this.forceInit = params.forceInit } getName() { @@ -50,7 +57,7 @@ export class ResolveProviderTask extends BaseTask { const rawProviderConfigs = keyBy(this.garden.getRawProviderConfigs(), "name") - return deps.map(providerName => { + return Bluebird.map(deps, async (providerName) => { const config = rawProviderConfigs[providerName] if (!config) { @@ -61,7 +68,7 @@ export class ResolveProviderTask extends BaseTask { ) } - const plugin = this.garden.getPlugin(providerName) + const plugin = await this.garden.getPlugin(providerName) return new ResolveProviderTask({ garden: this.garden, @@ -69,6 +76,7 @@ export class ResolveProviderTask extends BaseTask { config, log: this.log, version: this.version, + forceInit: this.forceInit, }) }) } @@ -77,11 +85,14 @@ export class ResolveProviderTask extends BaseTask { const resolvedProviders: Provider[] = Object.values(dependencyResults).map(result => result.output) const context = new ProviderConfigContext(this.garden.environmentName, this.garden.projectName, resolvedProviders) + + this.log.silly(`Resolving template strings for plugin ${this.config.name}`) let resolvedConfig = await resolveTemplateStrings(this.config, context) resolvedConfig.path = this.garden.projectRoot const providerName = resolvedConfig.name + this.log.silly(`Validating ${providerName} config`) if (this.plugin.configSchema) { resolvedConfig = validateWithPath({ config: resolvedConfig, @@ -115,6 +126,61 @@ export class ResolveProviderTask extends BaseTask { } } - return providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs) + this.log.silly(`Ensuring ${providerName} provider is ready`) + const tmpProvider = providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, defaultEnvironmentStatus) + const status = await this.ensurePrepared(tmpProvider) + + return providerFromConfig(resolvedConfig, resolvedProviders, moduleConfigs, status) + } + + private async ensurePrepared(tmpProvider: Provider) { + const pluginName = tmpProvider.name + const actions = await this.garden.getActionHelper() + const ctx = createPluginContext(this.garden, tmpProvider) + + const log = this.log.placeholder() + + this.log.silly(`Getting status for ${pluginName}`) + + const handler = await actions.getActionHandler({ + actionType: "getEnvironmentStatus", + pluginName, + defaultHandler: async () => defaultEnvironmentStatus, + }) + + let status = await handler({ ctx, log }) + + this.log.silly(`${pluginName} status: ${status.ready ? "ready" : "not ready"}`) + + if (this.forceInit || !status.ready) { + // Deliberately setting the text on the parent log here + this.log.setState(`Preparing environment...`) + + const envLogEntry = log.info({ + status: "active", + section: pluginName, + msg: "Configuring...", + }) + + const prepareHandler = await actions.getActionHandler({ + actionType: "prepareEnvironment", + pluginName, + defaultHandler: async () => ({ status }), + }) + const result = await prepareHandler({ ctx, log, force: this.forceInit, status }) + + status = result.status + + envLogEntry.setSuccess({ msg: chalk.green("Ready"), append: true }) + } + + if (!status.ready) { + throw new PluginError( + `Provider ${pluginName} reports status as not ready and could not prepare the configured environment.`, + { name: pluginName, status, provider: tmpProvider }, + ) + } + + return status } } diff --git a/garden-service/src/types/plugin/plugin.ts b/garden-service/src/types/plugin/plugin.ts index aeabf88931..8b38ecabbb 100644 --- a/garden-service/src/types/plugin/plugin.ts +++ b/garden-service/src/types/plugin/plugin.ts @@ -200,8 +200,9 @@ export const pluginActionNames: PluginActionName[] = Object. export const moduleActionNames: ModuleActionName[] = Object.keys(moduleActionDescriptions) export interface GardenPlugin { - configSchema?: Joi.Schema, + configSchema?: Joi.ObjectSchema, configKeys?: string[] + outputsSchema?: Joi.ObjectSchema, dependencies?: string[] @@ -228,6 +229,7 @@ export const pluginSchema = joi.object() .keys({ // TODO: make this a JSON/OpenAPI schema for portability configSchema: joi.object({ isJoi: joi.boolean().only(true).required() }).unknown(true), + outputsSchema: joi.object({ isJoi: joi.boolean().only(true).required() }).unknown(true), dependencies: joiArray(joi.string()) .description(deline` Names of plugins that need to be configured prior to this plugin. This plugin will be able to reference the diff --git a/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts b/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts index 6b5449cb2f..ce88bb023b 100644 --- a/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts +++ b/garden-service/src/types/plugin/provider/getEnvironmentStatus.ts @@ -6,35 +6,26 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -import { DashboardPage, dashboardPagesSchema } from "../../../config/dashboard" import { PluginActionParamsBase, actionParamsSchema } from "../base" import { dedent } from "../../../util/string" -import { joi } from "../../../config/common" +import { PrimitiveMap } from "../../../config/common" +import { DashboardPage, environmentStatusSchema } from "../../../config/status" export interface GetEnvironmentStatusParams extends PluginActionParamsBase { } -export interface EnvironmentStatus { +export interface EnvironmentStatus { ready: boolean dashboardPages?: DashboardPage[] detail?: any + outputs: T } +export const defaultEnvironmentStatus: EnvironmentStatus = { ready: true, outputs: {} } + export interface EnvironmentStatusMap { [providerName: string]: EnvironmentStatus } -export const environmentStatusSchema = joi.object() - .keys({ - ready: joi.boolean() - .required() - .description("Set to true if the environment is fully configured for a provider."), - dashboardPages: dashboardPagesSchema, - detail: joi.object() - .meta({ extendable: true }) - .description("Use this to include additional information that is specific to the provider."), - }) - .description("Description of an environment's status for a provider.") - export const getEnvironmentStatus = { description: dedent` Check if the current environment is ready for use by this plugin. Use this action in combination diff --git a/garden-service/src/types/plugin/provider/prepareEnvironment.ts b/garden-service/src/types/plugin/provider/prepareEnvironment.ts index 410f1b4905..3d66acfb38 100644 --- a/garden-service/src/types/plugin/provider/prepareEnvironment.ts +++ b/garden-service/src/types/plugin/provider/prepareEnvironment.ts @@ -7,16 +7,19 @@ */ import { PluginActionParamsBase, actionParamsSchema } from "../base" -import { environmentStatusSchema, EnvironmentStatus } from "./getEnvironmentStatus" +import { EnvironmentStatus } from "./getEnvironmentStatus" import { dedent } from "../../../util/string" import { joi } from "../../../config/common" +import { environmentStatusSchema } from "../../../config/status" export interface PrepareEnvironmentParams extends PluginActionParamsBase { status: EnvironmentStatus force: boolean } -export interface PrepareEnvironmentResult { } +export interface PrepareEnvironmentResult { + status: EnvironmentStatus, +} export const prepareEnvironment = { description: dedent` @@ -32,5 +35,8 @@ export const prepareEnvironment = { .description("Force re-configuration of the environment."), status: environmentStatusSchema, }), - resultSchema: joi.object().keys({}), + resultSchema: joi.object() + .keys({ + status: environmentStatusSchema, + }), } diff --git a/garden-service/test/e2e/garden.yml b/garden-service/test/e2e/garden.yml index 2ddcae63c6..73e9333e81 100644 --- a/garden-service/test/e2e/garden.yml +++ b/garden-service/test/e2e/garden.yml @@ -22,9 +22,8 @@ tests: command: [npm, run, e2e-full, --, --project=tasks, --showlog=true, --env=testing] - name: hot-reload # Tests for hot-reload are currently being skipped command: [npm, run, e2e-full, --, --project=hot-reload, --showlog=true, --env=testing] - # Disabling until https://github.com/garden-io/garden/issues/1045 is fixed - # - name: openfaas - # command: [npm, run, e2e-full, --, --project=openfaas, --showlog=true, --env=testing] + - name: openfaas + command: [npm, run, e2e-full, --, --project=openfaas, --showlog=true, --env=testing] - name: project-variables command: [npm, run, e2e-full, --, --project=project-variables, --showlog=true, --env=testing] - name: vote-helm diff --git a/garden-service/test/helpers.ts b/garden-service/test/helpers.ts index 305146e3f1..40532bbe38 100644 --- a/garden-service/test/helpers.ts +++ b/garden-service/test/helpers.ts @@ -129,7 +129,7 @@ export const testPlugin: PluginFactory = (): GardenPlugin => { return { actions: { async prepareEnvironment() { - return {} + return { status: { ready: true, outputs: {} } } }, async setSecret({ key, value }: SetSecretParams) { diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile b/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile index ad14ad92fb..67ea36f259 100644 --- a/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile +++ b/garden-service/test/unit/data/test-projects/helm/api-image/Dockerfile @@ -1,18 +1 @@ -# Using official python runtime base image -FROM python:2.7-alpine - -# Set the application directory -WORKDIR /app - -# Install our requirements.txt -ADD requirements.txt /app/requirements.txt -RUN pip install -r requirements.txt - -# Copy our code from the current folder to /app inside the container -ADD . /app - -# Make port 80 available for links and/or publish -EXPOSE 80 - -# Define our command to be run when launching the container -CMD ["gunicorn", "app:app", "-b", "0.0.0.0:80", "--log-file", "-", "--access-logfile", "-", "--workers", "4", "--keep-alive", "0"] +FROM busybox:1.31.0 \ No newline at end of file diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/app.py b/garden-service/test/unit/data/test-projects/helm/api-image/app.py deleted file mode 100644 index 6db2b765ee..0000000000 --- a/garden-service/test/unit/data/test-projects/helm/api-image/app.py +++ /dev/null @@ -1,52 +0,0 @@ -from flask import Flask, render_template, request, make_response, g -from flask_cors import CORS -from redis import Redis -import os -import socket -import random -import json - -option_a = os.getenv('OPTION_A', "Cats") -option_b = os.getenv('OPTION_B', "Dogs") -hostname = socket.gethostname() - -app = Flask(__name__) -CORS(app) - -def get_redis(): - if not hasattr(g, 'redis'): - g.redis = Redis(host="redis-master", db=0, socket_timeout=5) - return g.redis - -@app.route("/vote/", methods=['POST','GET']) -def vote(): - voter_id = hex(random.getrandbits(64))[2:-1] - - app.logger.info("received request") - - vote = None - - if request.method == 'POST': - redis = get_redis() - vote = request.form['vote'] - data = json.dumps({'voter_id': voter_id, 'vote': vote}) - - redis.rpush('votes', data) - print("Registered vote") - response = app.response_class( - response=json.dumps(data), - status=200, - mimetype='application/json' - ) - return response - - response = app.response_class( - response=json.dumps({}), - status=404, - mimetype='application/json' - ) - return response - - -if __name__ == "__main__": - app.run(host='0.0.0.0', port=80, debug=True, threaded=True) diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml b/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml index d23ed371a8..c5122eb44f 100644 --- a/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml +++ b/garden-service/test/unit/data/test-projects/helm/api-image/garden.yml @@ -1,11 +1,8 @@ -module: - description: Image for the API backend for the voting UI - type: container - name: api-image - hotReload: - sync: - - source: "*" - target: /app - tests: - - name: unit - args: [echo, ok] +kind: Module +description: Image for the API backend for the voting UI +type: container +name: api-image +hotReload: + sync: + - source: "*" + target: /app \ No newline at end of file diff --git a/garden-service/test/unit/data/test-projects/helm/api-image/requirements.txt b/garden-service/test/unit/data/test-projects/helm/api-image/requirements.txt deleted file mode 100644 index dcd270a579..0000000000 --- a/garden-service/test/unit/data/test-projects/helm/api-image/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -Flask -Redis -gunicorn -flask-cors diff --git a/garden-service/test/unit/src/actions.ts b/garden-service/test/unit/src/actions.ts index d3dc858e7a..34571954a9 100644 --- a/garden-service/test/unit/src/actions.ts +++ b/garden-service/test/unit/src/actions.ts @@ -8,7 +8,6 @@ import { } from "../../../src/types/plugin/plugin" import { RuntimeContext, Service, getServiceRuntimeContext } from "../../../src/types/service" import { expectError, makeTestGardenA } from "../../helpers" - import { ActionHelper } from "../../../src/actions" import { Garden } from "../../../src/garden" import { LogEntry } from "../../../src/logger/log-entry" @@ -46,53 +45,38 @@ describe("ActionHelper", () => { // Note: The test plugins below implicitly validate input params for each of the tests describe("environment actions", () => { describe("getEnvironmentStatus", () => { - it("should return a map of statuses for providers that have a getEnvironmentStatus handler", async () => { - const result = await actions.getEnvironmentStatus({ log }) - expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, - "test-plugin-b": { ready: false, dashboardPages: [] }, - }) - }) - - it("should optionally filter to single plugin", async () => { + it("should return the environment status for a provider", async () => { const result = await actions.getEnvironmentStatus({ log, pluginName: "test-plugin" }) expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, + ready: false, + outputs: {}, + dashboardPages: [], }) }) }) describe("prepareEnvironment", () => { - it("should prepare the environment for each configured provider", async () => { - const result = await actions.prepareEnvironment({ log }) - expect(result).to.eql({ - "test-plugin": true, - "test-plugin-b": true, + it("should prepare the environment for a configured provider", async () => { + const result = await actions.prepareEnvironment({ + log, + pluginName: "test-plugin", + force: false, + status: { ready: true, outputs: {} }, }) - }) - - it("should optionally filter to single plugin", async () => { - const result = await actions.prepareEnvironment({ log, pluginName: "test-plugin" }) expect(result).to.eql({ - "test-plugin": true, + status: { + ready: true, + outputs: {}, + dashboardPages: [], + }, }) }) }) describe("cleanupEnvironment", () => { - it("should clean up environment for each configured provider", async () => { - const result = await actions.cleanupEnvironment({ log }) - expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, - "test-plugin-b": { ready: false, dashboardPages: [] }, - }) - }) - - it("should optionally filter to single plugin", async () => { + it("should clean up environment for a provider", async () => { const result = await actions.cleanupEnvironment({ log, pluginName: "test-plugin" }) - expect(result).to.eql({ - "test-plugin": { ready: false, dashboardPages: [] }, - }) + expect(result).to.eql({}) }) }) @@ -322,7 +306,7 @@ describe("ActionHelper", () => { describe("getActionHandlers", () => { it("should return all handlers for a type", async () => { - const handlers = actions.getActionHandlers("prepareEnvironment") + const handlers = await actions.getActionHandlers("prepareEnvironment") expect(Object.keys(handlers)).to.eql([ "test-plugin", @@ -333,7 +317,7 @@ describe("ActionHelper", () => { describe("getModuleActionHandlers", () => { it("should return all handlers for a type", async () => { - const handlers = actions.getModuleActionHandlers({ actionType: "build", moduleType: "exec" }) + const handlers = await actions.getModuleActionHandlers({ actionType: "build", moduleType: "exec" }) expect(Object.keys(handlers)).to.eql([ "exec", @@ -341,29 +325,22 @@ describe("ActionHelper", () => { }) }) - describe("getActionHelper", () => { - it("should return last configured handler for specified action type", async () => { + describe("getActionHandler", () => { + it("should return the configured handler for specified action type and plugin name", async () => { const gardenA = await makeTestGardenA() const actionsA = await gardenA.getActionHelper() - const handler = actionsA.getActionHelper({ actionType: "prepareEnvironment" }) + const pluginName = "test-plugin-b" + const handler = await actionsA.getActionHandler({ actionType: "prepareEnvironment", pluginName }) expect(handler["actionType"]).to.equal("prepareEnvironment") - expect(handler["pluginName"]).to.equal("test-plugin-b") - }) - - it("should optionally filter to only handlers for the specified module type", async () => { - const gardenA = await makeTestGardenA() - const actionsA = await gardenA.getActionHelper() - const handler = actionsA.getActionHelper({ actionType: "prepareEnvironment" }) - - expect(handler["actionType"]).to.equal("prepareEnvironment") - expect(handler["pluginName"]).to.equal("test-plugin-b") + expect(handler["pluginName"]).to.equal(pluginName) }) it("should throw if no handler is available", async () => { const gardenA = await makeTestGardenA() const actionsA = await gardenA.getActionHelper() - await expectError(() => actionsA.getActionHelper({ actionType: "cleanupEnvironment" }), "parameter") + const pluginName = "test-plugin-b" + await expectError(() => actionsA.getActionHandler({ actionType: "cleanupEnvironment", pluginName }), "plugin") }) }) @@ -371,7 +348,7 @@ describe("ActionHelper", () => { it("should return last configured handler for specified module action type", async () => { const gardenA = await makeTestGardenA() const actionsA = await gardenA.getActionHelper() - const handler = actionsA.getModuleActionHandler({ actionType: "deployService", moduleType: "test" }) + const handler = await actionsA.getModuleActionHandler({ actionType: "deployService", moduleType: "test" }) expect(handler["actionType"]).to.equal("deployService") expect(handler["pluginName"]).to.equal("test-plugin-b") @@ -394,12 +371,13 @@ const testPlugin: PluginFactory = async () => ({ validate(params, pluginActionDescriptions.getEnvironmentStatus.paramsSchema) return { ready: false, + outputs: {}, } }, prepareEnvironment: async (params) => { validate(params, pluginActionDescriptions.prepareEnvironment.paramsSchema) - return {} + return { status: { ready: true, outputs: {} } } }, cleanupEnvironment: async (params) => { diff --git a/garden-service/test/unit/src/commands/delete.ts b/garden-service/test/unit/src/commands/delete.ts index dcb4e6fe9d..17696b876b 100644 --- a/garden-service/test/unit/src/commands/delete.ts +++ b/garden-service/test/unit/src/commands/delete.ts @@ -70,12 +70,12 @@ describe("DeleteEnvironmentCommand", () => { const testEnvStatuses: { [key: string]: EnvironmentStatus } = {} const cleanupEnvironment = async () => { - testEnvStatuses[name] = { ready: false } + testEnvStatuses[name] = { ready: false, outputs: {} } return {} } const getEnvironmentStatus = async () => { - return testEnvStatuses[name] + return testEnvStatuses[name] || { ready: true, outputs: {} } } const deleteService = async ({ service }): Promise => { diff --git a/garden-service/test/unit/src/commands/scan.ts b/garden-service/test/unit/src/commands/scan.ts index 737d045f7c..a17704fc5a 100644 --- a/garden-service/test/unit/src/commands/scan.ts +++ b/garden-service/test/unit/src/commands/scan.ts @@ -1,22 +1,19 @@ -import { Garden } from "../../../../src/garden" import { ScanCommand } from "../../../../src/commands/scan" -import { getExampleProjects, withDefaultGlobalOpts } from "../../../helpers" +import { withDefaultGlobalOpts, makeTestGardenA } from "../../../helpers" describe("ScanCommand", () => { - for (const [name, path] of Object.entries(getExampleProjects())) { - it(`should successfully scan the ${name} project`, async () => { - const garden = await Garden.factory(path) - const log = garden.log - const command = new ScanCommand() + it(`should successfully scan a test project`, async () => { + const garden = await makeTestGardenA() + const log = garden.log + const command = new ScanCommand() - await command.action({ - garden, - log, - headerLog: log, - footerLog: log, - args: {}, - opts: withDefaultGlobalOpts({}), - }) + await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: {}, + opts: withDefaultGlobalOpts({}), }) - } + }) }) diff --git a/garden-service/test/unit/src/commands/validate.ts b/garden-service/test/unit/src/commands/validate.ts index d7bb345ec4..2d0ba362c2 100644 --- a/garden-service/test/unit/src/commands/validate.ts +++ b/garden-service/test/unit/src/commands/validate.ts @@ -1,26 +1,23 @@ import { join } from "path" import { Garden } from "../../../../src/garden" import { ValidateCommand } from "../../../../src/commands/validate" -import { expectError, getExampleProjects, withDefaultGlobalOpts, dataDir } from "../../../helpers" +import { expectError, withDefaultGlobalOpts, dataDir, makeTestGardenA } from "../../../helpers" describe("commands.validate", () => { - // validate all of the example projects - for (const [name, path] of Object.entries(getExampleProjects())) { - it(`should successfully validate the ${name} project`, async () => { - const garden = await Garden.factory(path) - const log = garden.log - const command = new ValidateCommand() + it(`should successfully validate a test project`, async () => { + const garden = await makeTestGardenA() + const log = garden.log + const command = new ValidateCommand() - await command.action({ - garden, - log, - headerLog: log, - footerLog: log, - args: {}, - opts: withDefaultGlobalOpts({}), - }) + await command.action({ + garden, + log, + headerLog: log, + footerLog: log, + args: {}, + opts: withDefaultGlobalOpts({}), }) - } + }) it("should fail validating the bad-project project", async () => { const root = join(dataDir, "validate", "bad-project") diff --git a/garden-service/test/unit/src/config/provider.ts b/garden-service/test/unit/src/config/provider.ts index 806f32459e..bfe2c4276f 100644 --- a/garden-service/test/unit/src/config/provider.ts +++ b/garden-service/test/unit/src/config/provider.ts @@ -9,8 +9,8 @@ describe("getProviderDependencies", () => { it("should extract implicit provider dependencies from template strings", async () => { const config: ProviderConfig = { name: "my-provider", - someKey: "\${provider.other-provider.foo}", - anotherKey: "foo-\${provider.another-provider.bar}", + someKey: "\${providers.other-provider.foo}", + anotherKey: "foo-\${providers.another-provider.bar}", } expect(await getProviderDependencies(plugin, config)).to.eql([ "another-provider", @@ -21,7 +21,7 @@ describe("getProviderDependencies", () => { it("should ignore template strings that don't reference providers", async () => { const config: ProviderConfig = { name: "my-provider", - someKey: "\${provider.other-provider.foo}", + someKey: "\${providers.other-provider.foo}", anotherKey: "foo-\${some.other.ref}", } expect(await getProviderDependencies(plugin, config)).to.eql([ @@ -32,15 +32,15 @@ describe("getProviderDependencies", () => { it("should throw on provider-scoped template strings without a provider name", async () => { const config: ProviderConfig = { name: "my-provider", - someKey: "\${provider}", + someKey: "\${providers}", } await expectError( () => getProviderDependencies(plugin, config), (err) => { expect(err.message).to.equal( - "Invalid template key 'provider' in configuration for provider 'my-provider'. " + - "You must specify a provider name as well (e.g. \\\${provider.my-provider}).", + "Invalid template key 'providers' in configuration for provider 'my-provider'. " + + "You must specify a provider name as well (e.g. \\\${providers.my-provider}).", ) }, ) diff --git a/garden-service/test/unit/src/garden.ts b/garden-service/test/unit/src/garden.ts index 0c90240a84..99d8d5284b 100644 --- a/garden-service/test/unit/src/garden.ts +++ b/garden-service/test/unit/src/garden.ts @@ -76,6 +76,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "container", @@ -85,6 +89,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin", @@ -95,6 +103,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin-b", @@ -105,6 +117,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, ]) @@ -133,6 +149,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "container", @@ -142,6 +162,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin", @@ -152,6 +176,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, ]) @@ -235,6 +263,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "container", @@ -244,6 +276,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin", @@ -254,6 +290,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, { name: "test-plugin-b", @@ -264,6 +304,10 @@ describe("Garden", () => { }, dependencies: [], moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, }, ]) }) @@ -522,8 +566,8 @@ describe("Garden", () => { { name: "default", variables: {} }, ], providers: [ - { name: "test-a", foo: "\${provider.test-b.outputs.foo}" }, - { name: "test-b", foo: "\${provider.test-a.outputs.foo}" }, + { name: "test-a", foo: "\${providers.test-b.outputs.foo}" }, + { name: "test-b", foo: "\${providers.test-a.outputs.foo}" }, ], variables: {}, } @@ -562,7 +606,7 @@ describe("Garden", () => { { name: "default", variables: {} }, ], providers: [ - { name: "test-a", foo: "\${provider.test-b.outputs.foo}" }, + { name: "test-a", foo: "\${providers.test-b.outputs.foo}" }, { name: "test-b" }, ], variables: {}, @@ -651,6 +695,49 @@ describe("Garden", () => { ), ) }) + + it("should allow providers to reference each others' outputs", async () => { + const testA: PluginFactory = (): GardenPlugin => { + return { + actions: { + getEnvironmentStatus: async () => { + return { + ready: true, + outputs: { foo: "bar" }, + } + }, + }, + } + } + + const testB: PluginFactory = (): GardenPlugin => { + return {} + } + + const projectConfig: ProjectConfig = { + apiVersion: "garden.io/v0", + kind: "Project", + name: "test", + path: projectRootA, + defaultEnvironment: "default", + dotIgnoreFiles: defaultDotIgnoreFiles, + environments: [ + { name: "default", variables: {} }, + ], + providers: [ + { name: "test-a" }, + { name: "test-b", foo: "\${providers.test-a.outputs.foo}" }, + ], + variables: {}, + } + + const plugins: Plugins = { "test-a": testA, "test-b": testB } + const garden = await TestGarden.factory(projectRootA, { config: projectConfig, plugins }) + + const providerB = await garden.resolveProvider("test-b") + + expect(providerB.config.foo).to.equal("bar") + }) }) describe("scanForConfigs", () => { diff --git a/garden-service/test/unit/src/plugins/container/container.ts b/garden-service/test/unit/src/plugins/container/container.ts index b54c6930f7..a9228617ff 100644 --- a/garden-service/test/unit/src/plugins/container/container.ts +++ b/garden-service/test/unit/src/plugins/container/container.ts @@ -64,7 +64,8 @@ describe("plugins.container", () => { beforeEach(async () => { garden = await makeTestGarden(projectRoot, { extraPlugins: { container: gardenPlugin } }) log = garden.log - ctx = await garden.getPluginContext("container") + const provider = await garden.resolveProvider("container") + ctx = await garden.getPluginContext(provider) td.replace(garden.buildDir, "syncDependencyProducts", () => null) diff --git a/garden-service/test/unit/src/plugins/container/helpers.ts b/garden-service/test/unit/src/plugins/container/helpers.ts index 9c0b4a08a0..6139c71040 100644 --- a/garden-service/test/unit/src/plugins/container/helpers.ts +++ b/garden-service/test/unit/src/plugins/container/helpers.ts @@ -60,7 +60,8 @@ describe("containerHelpers", () => { beforeEach(async () => { garden = await makeTestGarden(projectRoot, { extraPlugins: { container: gardenPlugin } }) log = garden.log - ctx = await garden.getPluginContext("container") + const provider = await garden.resolveProvider("container") + ctx = await garden.getPluginContext(provider) td.replace(garden.buildDir, "syncDependencyProducts", () => null) diff --git a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts index f185d42edd..1f3c72f155 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/container/ingress.ts @@ -57,6 +57,7 @@ const basicProvider: KubernetesProvider = { config: basicConfig, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const singleTlsConfig: KubernetesConfig = { @@ -76,6 +77,7 @@ const singleTlsProvider: KubernetesProvider = { config: singleTlsConfig, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const multiTlsConfig: KubernetesConfig = { @@ -111,6 +113,7 @@ const multiTlsProvider: KubernetesProvider = { config: multiTlsConfig, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } // generated with `openssl req -newkey rsa:2048 -nodes -keyout key.pem -x509 -days 365 -out certificate.pem` @@ -373,7 +376,8 @@ describe("createIngressResources", () => { testConfigs: [], } - const ctx = await garden.getPluginContext("container") + const provider = await garden.resolveProvider("container") + const ctx = await garden.getPluginContext(provider) const parsed = await configure({ ctx, moduleConfig, log: garden.log }) const graph = await garden.getConfigGraph() const module = await moduleFromConfig(garden, graph, parsed) @@ -639,6 +643,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const err: any = new Error("nope") @@ -668,6 +673,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const api = await getKubeApi(basicConfig.context) @@ -699,6 +705,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } const api = await getKubeApi(basicConfig.context) @@ -788,6 +795,7 @@ describe("createIngressResources", () => { }, dependencies: [], moduleConfigs: [], + status: { ready: true, outputs: {} }, } td.when(api.core.readNamespacedSecret("foo", "default")).thenResolve(myDomainCertSecret) diff --git a/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts b/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts index 3e9adba2c6..afd72d3488 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/helm/common.ts @@ -1,7 +1,7 @@ import { TestGarden, makeTestGarden, dataDir, expectError } from "../../../../../helpers" import { resolve } from "path" import { expect } from "chai" -import { first } from "lodash" +import { first, set } from "lodash" import { containsSource, @@ -21,6 +21,29 @@ import { deline } from "../../../../../../src/util/string" import { HotReloadableResource } from "../../../../../../src/plugins/kubernetes/hot-reload" import { getServiceResourceSpec } from "../../../../../../src/plugins/kubernetes/helm/common" import { ConfigGraph } from "../../../../../../src/config-graph" +import { Provider } from "../../../../../../src/config/provider" + +const helmProvider: Provider = { + name: "local-kubernetes", + config: { + name: "local-kubernetes", + buildMode: "local-docker", + }, + dependencies: [], + moduleConfigs: [], + status: { + ready: true, + outputs: {}, + }, +} + +export async function getHelmTestGarden() { + const projectRoot = resolve(dataDir, "test-projects", "helm") + const garden = await makeTestGarden(projectRoot) + // Avoid having to resolve the provider + set(garden, "resolvedProviders", [helmProvider]) + return garden +} describe("Helm common functions", () => { let garden: TestGarden @@ -29,10 +52,11 @@ describe("Helm common functions", () => { let log: LogEntry before(async () => { - const projectRoot = resolve(dataDir, "test-projects", "helm") - garden = await makeTestGarden(projectRoot) + garden = await getHelmTestGarden() + // Avoid having to resolve the provider + set(garden, "resolvedProviders", [helmProvider]) graph = await garden.getConfigGraph() - ctx = await garden.getPluginContext("local-kubernetes") + ctx = await garden.getPluginContext(helmProvider) log = garden.log await buildModules() }) @@ -46,7 +70,7 @@ describe("Helm common functions", () => { const tasks = modules.map(module => new BuildTask({ garden, log, module, force: false })) const results = await garden.processTasks(tasks) - const err = first(Object.values(results).map(r => r.error)) + const err = first(Object.values(results).map(r => r && r.error)) if (err) { throw err @@ -68,7 +92,6 @@ describe("Helm common functions", () => { describe("getChartResources", () => { it("should render and return resources for a local template", async () => { const module = await graph.getModule("api") - const imageModule = await graph.getModule("api-image") const resources = await getChartResources(ctx, module, log) expect(resources).to.eql([ @@ -133,7 +156,7 @@ describe("Helm common functions", () => { containers: [ { name: "api", - image: "api-image:" + imageModule.version.versionString, + image: resources[1].spec.template.spec.containers[0].image, imagePullPolicy: "IfNotPresent", args: [ "python", diff --git a/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts b/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts index 4a05b7a2aa..a2e423f6ff 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/helm/config.ts @@ -2,11 +2,12 @@ import { resolve } from "path" import { expect } from "chai" import { cloneDeep } from "lodash" -import { TestGarden, dataDir, makeTestGarden, expectError } from "../../../../../helpers" +import { TestGarden, expectError } from "../../../../../helpers" import { PluginContext } from "../../../../../../src/plugin-context" import { deline } from "../../../../../../src/util/string" import { ModuleConfig } from "../../../../../../src/config/module" import { apply } from "json-merge-patch" +import { getHelmTestGarden } from "./common" describe("validateHelmModule", () => { let garden: TestGarden @@ -14,9 +15,9 @@ describe("validateHelmModule", () => { let moduleConfigs: { [key: string]: ModuleConfig } before(async () => { - const projectRoot = resolve(dataDir, "test-projects", "helm") - garden = await makeTestGarden(projectRoot) - ctx = await garden.getPluginContext("local-kubernetes") + garden = await getHelmTestGarden() + const provider = await garden.resolveProvider("local-kubernetes") + ctx = await garden.getPluginContext(provider) await garden.resolveModuleConfigs() moduleConfigs = cloneDeep((garden).moduleConfigs) }) diff --git a/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts b/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts index d1d2764d6c..ed85f4df1f 100644 --- a/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts +++ b/garden-service/test/unit/src/plugins/kubernetes/helm/hot-reload.ts @@ -1,18 +1,17 @@ -import { resolve } from "path" import { expect } from "chai" -import { dataDir, makeTestGarden, TestGarden, expectError } from "../../../../../helpers" +import { TestGarden, expectError } from "../../../../../helpers" import { getHotReloadSpec } from "../../../../../../src/plugins/kubernetes/helm/hot-reload" import { deline } from "../../../../../../src/util/string" import { ConfigGraph } from "../../../../../../src/config-graph" +import { getHelmTestGarden } from "./common" describe("getHotReloadSpec", () => { let garden: TestGarden let graph: ConfigGraph before(async () => { - const projectRoot = resolve(dataDir, "test-projects", "helm") - garden = await makeTestGarden(projectRoot) + garden = await getHelmTestGarden() graph = await garden.getConfigGraph() })