fix(k8s): hot reloading stopped working after config reload

Fixes #1033
garden-io · Jul 30, 2019 · a914d4b · a914d4b
1 parent e049579
commit a914d4b
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 6 deletions.
diff --git a/garden-service/src/plugins/kubernetes/hot-reload.ts b/garden-service/src/plugins/kubernetes/hot-reload.ts
@@ -19,7 +19,7 @@ import { Service } from "../../types/service"
 import { LogEntry } from "../../logger/log-entry"
 import { getResourceContainer } from "./helm/common"
 import { waitForContainerService } from "./container/status"
-import { getPortForward } from "./util"
+import { getPortForward, killPortForward } from "./util"
 import { RSYNC_PORT } from "./constants"
 import { getAppNamespace } from "./namespace"
 import { KubernetesPluginContext } from "./config"
@@ -244,7 +244,7 @@ export async function syncToService(
   const targetDeployment = `${targetKind.toLowerCase()}/${targetName}`
   const namespace = await getAppNamespace(ctx, log, ctx.provider)
 
-  try {
+  const doSync = async () => {
     const portForward = await getPortForward({ ctx, log, namespace, targetDeployment, port: RSYNC_PORT })
 
     return Bluebird.map(hotReloadSpec.sync, ({ source, target }) => {
@@ -255,6 +255,20 @@ export async function syncToService(
 
       return execa("rsync", ["-vrpztgo", src, destination])
     })
+  }
+
+  try {
+    try {
+      await doSync()
+    } catch (error) {
+      if (error.message.includes("did not see server greeting")) {
+        log.debug(`Port-forward to ${targetDeployment} disconnected. Retrying.`)
+        killPortForward(targetDeployment, RSYNC_PORT)
+        await doSync()
+      } else {
+        throw error
+      }
+    }
   } catch (error) {
     throw new RuntimeError(`Unexpected error while synchronising to service ${service.name}: ${error.message}`, {
       error,

diff --git a/garden-service/src/plugins/kubernetes/util.ts b/garden-service/src/plugins/kubernetes/util.ts
@@ -114,6 +114,8 @@ export function deduplicateResources(resources: KubernetesResource[]) {
 }
 
 export interface PortForward {
+  targetDeployment: string
+  port: number
   localPort: number
   proc: ChildProcess
 }
@@ -122,17 +124,30 @@ const registeredPortForwards: { [key: string]: PortForward } = {}
 const portForwardRegistrationLock = new AsyncLock()
 
 registerCleanupFunction("kill-port-forward-procs", () => {
-  for (const { proc } of Object.values(registeredPortForwards)) {
-    !proc.killed && proc.kill()
+  for (const { targetDeployment, port } of Object.values(registeredPortForwards)) {
+    killPortForward(targetDeployment, port)
   }
 })
 
+export function killPortForward(targetDeployment: string, port: number) {
+  const key = getPortForwardKey(targetDeployment, port)
+  const fwd = registeredPortForwards[key]
+  if (fwd) {
+    const { proc } = fwd
+    !proc.killed && proc.kill()
+  }
+}
+
+function getPortForwardKey(targetDeployment: string, port: number) {
+  return `${targetDeployment}:${port}`
+}
+
 export async function getPortForward(
   { ctx, log, namespace, targetDeployment, port }:
     { ctx: PluginContext, log: LogEntry, namespace: string, targetDeployment: string, port: number },
 ): Promise<PortForward> {
   // Using lock here to avoid concurrency issues (multiple parallel requests for same forward).
-  const key = `${targetDeployment}:${port}`
+  const key = getPortForwardKey(targetDeployment, port)
 
   return portForwardRegistrationLock.acquire("register-port-forward", (async () => {
     let localPort: number
@@ -169,7 +184,7 @@ export async function getPortForward(
         log.silly(`[${targetDeployment} port forwarder] ${line}`)
 
         if (line.toString().includes("Forwarding from ")) {
-          const portForward = { proc, localPort }
+          const portForward = { targetDeployment, port, proc, localPort }
           registeredPortForwards[key] = portForward
           resolve(portForward)
         }