fix(k8s): retry websocket errors

Fixes #5729

We failed retrying WebsocketError during Pod exec operations, because the regex "WebsocketError: Unexpected server response" did not match
anymore after we refactored toKubernetesError, because the formatting
cahnged.

Let's retry if just the name WebsocketError appears in the message.

core/package.json

@@ -40,6 +40,7 @@
     "@opentelemetry/sdk-trace-base": "^1.21.0",
     "@opentelemetry/semantic-conventions": "^1.17.1",
     "@scg82/exit-hook": "^3.4.1",
+    "@types/ws": "^8.5.10",
     "ajv": "^8.12.0",
     "ajv-formats": "^2.1.1",
     "analytics-node": "6.2.0",

core/src/plugins/kubernetes/retry.ts

@@ -201,8 +201,10 @@ const errorMessageRegexesForRetry = [
   // (rpc error: code = ResourceExhausted desc = etcdserver: throttle: too many requests)
   /too many requests/,
   /Unable to connect to the server/,
-  /WebsocketError: Unexpected server response/,
   // We often flaked with this error on microk8s in the CI:
   // > pods "api-test-xxxx" is forbidden: error looking up service account container-default/default: serviceaccount "default" not found
   /forbidden: error looking up service account/,
+
+  // We get WebsocketError without HTTP status code on some API operations, e.g. exec in a pod
+  /WebsocketError/,
 ]

core/test/unit/src/plugins/kubernetes/retry.ts

@@ -0,0 +1,42 @@
+/*
+ * Copyright (C) 2018-2023 Garden Technologies, Inc. <info@garden.io>
+ *
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+import { KubernetesError } from "../../../../../src/plugins/kubernetes/api.js"
+import type { ErrorEvent, WebSocket } from "ws"
+import { shouldRetry, toKubernetesError } from "../../../../../src/plugins/kubernetes/retry.js"
+import { expect } from "chai"
+import dedent from "dedent"
+
+const testKubeOp = "test"
+const websocketError: ErrorEvent = {
+  error: new Error("error message"),
+  message: "This is a test error message",
+  type: "error",
+  target: true as unknown as WebSocket,
+}
+
+describe("toKubernetesError", () => {
+  it("should handle WebsocketError", () => {
+    const err = toKubernetesError(websocketError, testKubeOp)
+
+    expect(err).to.be.instanceof(KubernetesError)
+    expect(err.message).to.equal(dedent`
+      Error while performing Kubernetes API operation test: WebsocketError
+
+      This is a test error message
+    `)
+    expect(err.responseStatusCode).to.be.undefined
+    expect(err.apiMessage).to.be.undefined
+    expect(err.type).to.equal("kubernetes")
+  })
+})
+
+describe("shouldRetry", () => {
+  it("should retry WebsocketError", () => {
+    expect(shouldRetry(websocketError, testKubeOp)).to.be.true
+  })
+})