feat(k8s): add timeout parameter to kubernetes module type

The default deployment timeout of 300 seconds could be too short,
best to let users configure that themselves if needed.

.circleci/config.yml

@@ -673,7 +673,7 @@ jobs:
       - run:
           name: Integ tests
           # Note: We skip tests that only work for remote environments
-          command: cd core && yarn run integ-local -b
+          command: cd core && yarn run integ-local
       - run:
           name: Plugin tests
           command: yarn run test:plugins

core/src/plugins/kubernetes/container/deployment.ts

@@ -14,7 +14,7 @@ import { ContainerModule, ContainerService, ContainerVolumeSpec, ContainerServic
 import { createIngressResources } from "./ingress"
 import { createServiceResources } from "./service"
 import { waitForResources, compareDeployedResources } from "../status/status"
-import { apply, deleteObjectsBySelector } from "../kubectl"
+import { apply, deleteObjectsBySelector, KUBECTL_DEFAULT_TIMEOUT } from "../kubectl"
 import { getAppNamespace, getAppNamespaceStatus } from "../namespace"
 import { PluginContext } from "../../../plugin-context"
 import { KubeApi } from "../api"
@@ -132,6 +132,7 @@ export async function deployContainerServiceRolling(params: DeployServiceParams<
     serviceName: service.name,
     resources: manifests,
     log,
+    timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
   })
 }
 
@@ -174,6 +175,7 @@ export async function deployContainerServiceBlueGreen(params: DeployServiceParam
       serviceName: service.name,
       resources: manifests,
       log,
+      timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
     })
   } else {
     // A k8s service matching the current Garden service exist in the cluster.
@@ -193,6 +195,7 @@ export async function deployContainerServiceBlueGreen(params: DeployServiceParam
       serviceName: `Deploy ${service.name}`,
       resources: filteredManifests,
       log,
+      timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
     })
 
     // Patch for the current service to point to the new Deployment
@@ -232,6 +235,7 @@ export async function deployContainerServiceBlueGreen(params: DeployServiceParam
       serviceName: `Update service`,
       resources: [serviceManifest],
       log,
+      timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
     })
 
     // Clenup unused deployments:

core/src/plugins/kubernetes/container/status.ts

@@ -103,7 +103,8 @@ export async function waitForContainerService(
   runtimeContext: RuntimeContext,
   service: GardenService,
   devMode: boolean,
-  hotReload: boolean
+  hotReload: boolean,
+  timeout = KUBECTL_DEFAULT_TIMEOUT
 ) {
   const startTime = new Date().getTime()
 
@@ -124,8 +125,8 @@ export async function waitForContainerService(
 
     log.silly(`Waiting for service ${service.name}`)
 
-    if (new Date().getTime() - startTime > KUBECTL_DEFAULT_TIMEOUT * 1000) {
-      throw new DeploymentError(`Timed out waiting for service ${service.name} to deploy`, {
+    if (new Date().getTime() - startTime > timeout * 1000) {
+      throw new DeploymentError(`Timed out waiting for service ${service.name} to deploy after ${timeout} seconds`, {
         serviceName: service.name,
         status,
       })

core/src/plugins/kubernetes/helm/deployment.ts

@@ -124,6 +124,7 @@ export async function deployHelmService({
     serviceName: service.name,
     resources: manifests,
     log,
+    timeoutSec: module.spec.timeout,
   })
 
   const forwardablePorts = getForwardablePorts(manifests)

core/src/plugins/kubernetes/kubernetes-module/config.ts

@@ -27,6 +27,7 @@ import {
 } from "../config"
 import { ContainerModule } from "../../container/config"
 import { kubernetesDevModeSchema, KubernetesDevModeSpec } from "../dev-mode"
+import { KUBECTL_DEFAULT_TIMEOUT } from "../kubectl"
 
 // A Kubernetes Module always maps to a single Service
 export type KubernetesModuleSpec = KubernetesServiceSpec
@@ -44,6 +45,7 @@ export interface KubernetesServiceSpec {
   serviceResource?: ServiceResourceSpec
   tasks: KubernetesTaskSpec[]
   tests: KubernetesTestSpec[]
+  timeout?: number
 }
 
 export type KubernetesService = GardenService<KubernetesModule, ContainerModule>
@@ -95,6 +97,10 @@ export const kubernetesModuleSpecSchema = () =>
       }),
     tasks: joiSparseArray(kubernetesTaskSchema()),
     tests: joiSparseArray(kubernetesTestSchema()),
+    timeout: joi
+      .number()
+      .default(KUBECTL_DEFAULT_TIMEOUT)
+      .description("The maximum duration (in seconds) to wait for resources to deploy and become healthy."),
   })
 
 export async function configureKubernetesModule({

core/src/plugins/kubernetes/kubernetes-module/handlers.ts

@@ -27,7 +27,7 @@ import { configureDevMode, startDevModeSync } from "../dev-mode"
 import { HelmService } from "../helm/config"
 import { configureHotReload, getHotReloadContainerName, getHotReloadSpec } from "../hot-reload/helpers"
 import { HotReloadableResource, hotReloadK8s } from "../hot-reload/hot-reload"
-import { apply, deleteObjectsBySelector } from "../kubectl"
+import { apply, deleteObjectsBySelector, KUBECTL_DEFAULT_TIMEOUT } from "../kubectl"
 import { streamK8sLogs } from "../logs"
 import { getModuleNamespace, getModuleNamespaceStatus } from "../namespace"
 import { getForwardablePorts, getPortForwardHandler, killPortForwards } from "../port-forward"
@@ -167,6 +167,7 @@ export async function deployKubernetesService(
       serviceName: service.name,
       resources: namespaceManifests,
       log,
+      timeoutSec: service.spec.timeout || KUBECTL_DEFAULT_TIMEOUT,
     })
   }
 
@@ -194,6 +195,7 @@ export async function deployKubernetesService(
       serviceName: service.name,
       resources: prepareResult.manifests,
       log,
+      timeoutSec: service.spec.timeout || KUBECTL_DEFAULT_TIMEOUT,
     })
   }
 

core/src/plugins/kubernetes/run.ts

@@ -41,6 +41,7 @@ import { PluginContext } from "../../plugin-context"
 import { waitForResources, ResourceStatus } from "./status/status"
 import { RuntimeContext } from "../../runtime-context"
 import { getResourceRequirements } from "./container/util"
+import { KUBECTL_DEFAULT_TIMEOUT } from "./kubectl"
 
 // Default timeout for individual run/exec operations
 const defaultTimeout = 600
@@ -880,7 +881,7 @@ export class PodRunner extends PodRunnerParams {
    * commands in the same Pod. Note that you *must manually call `stop()`* when you're done. Otherwise the Pod will
    * stay running in the cluster until the process exits.
    */
-  async start({ log, timeoutSec }: StartParams) {
+  async start({ log, timeoutSec = KUBECTL_DEFAULT_TIMEOUT }: StartParams) {
     const { ctx, provider, pod, namespace } = this
 
     await this.createPod({ log, tty: false })

core/src/plugins/kubernetes/status/status.ts

@@ -12,7 +12,6 @@ import { PluginContext } from "../../../plugin-context"
 import { ServiceState, combineStates } from "../../../types/service"
 import { sleep, deepMap } from "../../../util/util"
 import { KubeApi } from "../api"
-import { KUBECTL_DEFAULT_TIMEOUT } from "../kubectl"
 import { getAppNamespace } from "../namespace"
 import Bluebird from "bluebird"
 import { KubernetesResource, KubernetesServerResource, BaseResource } from "../types"
@@ -167,7 +166,7 @@ interface WaitParams {
   serviceName?: string
   resources: KubernetesResource[]
   log: LogEntry
-  timeoutSec?: number
+  timeoutSec: number
 }
 
 /**
@@ -186,10 +185,6 @@ export async function waitForResources({
   let lastMessage: string | undefined
   const startTime = new Date().getTime()
 
-  if (!timeoutSec) {
-    timeoutSec = KUBECTL_DEFAULT_TIMEOUT
-  }
-
   const statusLine = log.info({
     symbol: "info",
     section: serviceName,
@@ -244,7 +239,10 @@ export async function waitForResources({
     const now = new Date().getTime()
 
     if (now - startTime > timeoutSec * 1000) {
-      throw new DeploymentError(`Timed out waiting for ${serviceName || "resources"} to deploy`, { statuses })
+      throw new DeploymentError(
+        `Timed out waiting for ${serviceName || "resources"} to deploy after ${timeoutSec} seconds`,
+        { statuses }
+      )
     }
   }
 

core/src/plugins/openfaas/openfaas.ts

@@ -50,6 +50,7 @@ import { trim } from "lodash"
 import { getModuleTypeUrl, getGitHubUrl } from "../../docs/common"
 import { PluginContext } from "../../plugin-context"
 import { getK8sProvider } from "../kubernetes/util"
+import { KUBECTL_DEFAULT_TIMEOUT } from "../kubernetes/kubectl"
 
 const systemDir = join(STATIC_DIR, "openfaas", "system")
 const moduleTypeUrl = getModuleTypeUrl("openfaas")
@@ -313,6 +314,7 @@ async function deployService(params: DeployServiceParams<OpenFaasModule>): Promi
     serviceName: service.name,
     log,
     resources,
+    timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
   })
 
   // TODO: avoid duplicate work here

core/test/integ/src/plugins/kubernetes/api.ts

@@ -19,6 +19,7 @@ import { expect } from "chai"
 import { waitForResources } from "../../../../../src/plugins/kubernetes/status/status"
 import { PluginContext } from "../../../../../src/plugin-context"
 import { StringCollector } from "../../../../../src/util/util"
+import { KUBECTL_DEFAULT_TIMEOUT } from "../../../../../src/plugins/kubernetes/kubectl"
 
 describe("KubeApi", () => {
   let garden: Garden
@@ -98,7 +99,15 @@ describe("KubeApi", () => {
       const podName = pod.metadata.name
 
       await api.createPod(namespace, pod)
-      await waitForResources({ namespace, ctx, provider, serviceName: "exec-test", resources: [pod], log: garden.log })
+      await waitForResources({
+        namespace,
+        ctx,
+        provider,
+        serviceName: "exec-test",
+        resources: [pod],
+        log: garden.log,
+        timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
+      })
 
       try {
         const res = await api.execInPod({
@@ -123,7 +132,15 @@ describe("KubeApi", () => {
       const podName = pod.metadata.name
 
       await api.createPod(namespace, pod)
-      await waitForResources({ namespace, ctx, provider, serviceName: "exec-test", resources: [pod], log: garden.log })
+      await waitForResources({
+        namespace,
+        ctx,
+        provider,
+        serviceName: "exec-test",
+        resources: [pod],
+        log: garden.log,
+        timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
+      })
 
       try {
         const res = await api.execInPod({
@@ -148,7 +165,15 @@ describe("KubeApi", () => {
       const podName = pod.metadata.name
 
       await api.createPod(namespace, pod)
-      await waitForResources({ namespace, ctx, provider, serviceName: "exec-test", resources: [pod], log: garden.log })
+      await waitForResources({
+        namespace,
+        ctx,
+        provider,
+        serviceName: "exec-test",
+        resources: [pod],
+        log: garden.log,
+        timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
+      })
 
       try {
         const res = await api.execInPod({
@@ -186,7 +211,15 @@ describe("KubeApi", () => {
       const podName = pod.metadata.name
 
       await api.createPod(namespace, pod)
-      await waitForResources({ namespace, ctx, provider, serviceName: "exec-test", resources: [pod], log: garden.log })
+      await waitForResources({
+        namespace,
+        ctx,
+        provider,
+        serviceName: "exec-test",
+        resources: [pod],
+        log: garden.log,
+        timeoutSec: KUBECTL_DEFAULT_TIMEOUT,
+      })
 
       const stdout = new StringCollector()