feat(helm): move native Helm wait logic into an init container

src/server/lib/kubernetes/jobFactory.ts

@@ -180,6 +180,7 @@ export interface HelmJobConfig {
   };
   includeGitClone?: boolean;
   registryAuth?: RegistryAuthConfig;
+  initContainers?: any[];
 }
 
 export function createHelmJob(config: HelmJobConfig): V1Job {
@@ -221,6 +222,10 @@ export function createHelmJob(config: HelmJobConfig): V1Job {
     initContainers.push(createRegistryAuthInitContainer(config.registryAuth));
   }
 
+  if (config.initContainers?.length) {
+    initContainers.push(...config.initContainers);
+  }
+
   const containers = config.containers.map((container) => ({
     ...container,
     resources: container.resources || {

src/server/lib/nativeHelm/__tests__/helm.test.ts

@@ -16,6 +16,7 @@
 
 import { shouldUseNativeHelm, createHelmContainer, nativeHelmDeploy, generateHelmManifest } from '../helm';
 import * as nativeHelmUtils from '../utils';
+import yaml from 'js-yaml';
 import {
   determineChartType,
   constructHelmCommand,
@@ -1537,6 +1538,9 @@ services:
         deployableId: 99,
         deployable: {
           name: 'sample-service',
+          repository: {
+            fullName: 'example-org/example-repo',
+          },
           helm: helmConfig,
         },
         build: {
@@ -1633,7 +1637,7 @@ services:
         'sample-chart': {
           chart: {
             name: 'sample-chart',
-            repoUrl: 'https://charts.example.com',
+            repoUrl: 'oci://123456789012.dkr.ecr.us-west-2.amazonaws.com/sample-chart',
             version: '2.5.1',
             valueFiles: ['global-values.yaml'],
           },
@@ -1695,13 +1699,21 @@ services:
       } as any;
 
       const manifest = await generateHelmManifest(deploy, 'test-job', { namespace: 'testns' });
+      const parsed = yaml.load(manifest) as any;
+      const initContainerNames = parsed.spec.template.spec.initContainers.map((container: any) => container.name);
+      const helmContainer = parsed.spec.template.spec.containers.find(
+        (container: any) => container.name === 'helm-deploy'
+      );
 
       expect(manifest).toContain('image: registry.example.com/service-helm-runner:2.0.0');
-      expect(manifest).toContain("--post-renderer '/opt/bin/post-renderer'");
-      expect(manifest).toContain("--post-renderer-args '--service-override'");
-      expect(manifest).not.toContain("--post-renderer-args '--global'");
       expect(manifest).toContain('-f service-values.yaml');
       expect(manifest).not.toContain('-f global-values.yaml');
+      expect(helmContainer.args[0]).toContain("--post-renderer '/opt/bin/post-renderer'");
+      expect(helmContainer.args[0]).toContain("--post-renderer-args '--service-override'");
+      expect(helmContainer.args[0]).not.toContain("--post-renderer-args '--global'");
+      expect(initContainerNames[initContainerNames.length - 1]).toBe('wait-for-prior-deploys');
+      expect(initContainerNames).toContain('ecr-auth');
+      expect(initContainerNames.indexOf('ecr-auth')).toBeLessThan(initContainerNames.indexOf('wait-for-prior-deploys'));
     });
   });
 });

src/server/lib/nativeHelm/constants.ts

@@ -19,6 +19,8 @@ export const HELM_JOB_TIMEOUT_SECONDS = HELM_TIMEOUT_MINUTES * 60;
 export const STATIC_ENV_JOB_TTL_SECONDS = 86400; // 24 hours
 export const DEFAULT_HELM_VERSION = '3.12.0';
 export const HELM_IMAGE_PREFIX = 'alpine/helm';
+export const HELM_WAIT_IMAGE = 'bitnamilegacy/kubectl:1.30';
+export const HELM_WAIT_TIMEOUT_SECONDS = 900;
 
 export const REPO_MAPPINGS = {
   bitnami: 'https://charts.bitnami.com/bitnami',

src/server/lib/nativeHelm/helm.ts

@@ -44,7 +44,7 @@ import {
   validateHelmConfiguration,
 } from './utils';
 import { detectRegistryAuth, RegistryAuthConfig } from './registryAuth';
-import { HELM_IMAGE_PREFIX } from './constants';
+import { HELM_IMAGE_PREFIX, HELM_WAIT_IMAGE, HELM_WAIT_TIMEOUT_SECONDS } from './constants';
 import { buildDeployJobName } from 'server/lib/kubernetes/jobNames';
 import {
   createCloneScript,
@@ -127,6 +127,50 @@ export async function createHelmContainer(
   };
 }
 
+export function createWaitForPriorDeploysInitContainer(namespace: string, serviceName: string, jobName: string): any {
+  const script = [
+    'set -euo pipefail',
+    `echo "Checking for prior deploy jobs for service=${serviceName}"`,
+    `MY_CREATED=$(kubectl get job ${jobName} -n ${namespace} -o jsonpath='{.metadata.creationTimestamp}')`,
+    `WAIT_TIMEOUT=${HELM_WAIT_TIMEOUT_SECONDS}`,
+    'POLL_INTERVAL=10',
+    'wait_start=$(date +%s)',
+    'while true; do',
+    '  elapsed=$(( $(date +%s) - wait_start ))',
+    '  if [ "$elapsed" -ge "$WAIT_TIMEOUT" ]; then',
+    '    echo "ERROR: Timed out after ${WAIT_TIMEOUT}s waiting for prior deploy jobs to complete"',
+    '    exit 1',
+    '  fi',
+    '',
+    `  blocking_jobs=$(kubectl get jobs -n ${namespace} -l "service=${serviceName},app.kubernetes.io/name=native-helm" -o jsonpath='{range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.active}{"\\n"}{end}' | awk -v my_name="${jobName}" -v my_ts="$MY_CREATED" '`,
+    '    BEGIN { result = "" }',
+    '    $1 != my_name && (($2 < my_ts) || ($2 == my_ts && $1 < my_name)) && ($3 + 0) > 0 {',
+    '      result = result (result ? " " : "") $1',
+    '    }',
+    `    END { print result }')`,
+    '',
+    '  if [ -z "$blocking_jobs" ]; then',
+    '    echo "No prior deploy jobs in progress, proceeding"',
+    '    break',
+    '  fi',
+    '',
+    '  echo "Waiting for prior deploy jobs to complete: $blocking_jobs (${elapsed}s/${WAIT_TIMEOUT}s)"',
+    '  sleep $POLL_INTERVAL',
+    'done',
+  ].join('\n');
+
+  return {
+    name: 'wait-for-prior-deploys',
+    image: HELM_WAIT_IMAGE,
+    command: ['/bin/bash', '-c'],
+    args: [script],
+    resources: {
+      requests: { cpu: '100m', memory: '128Mi' },
+      limits: { cpu: '500m', memory: '512Mi' },
+    },
+  };
+}
+
 export async function generateHelmManifest(
   deploy: Deploy,
   jobName: string,
@@ -161,6 +205,7 @@ export async function generateHelmManifest(
   const registryAuth = detectRegistryAuth(chartRepoUrl);
   const helmImage = mergedHelmConfig.nativeHelm?.image;
   const postRenderer = mergedHelmConfig.nativeHelm?.postRenderer;
+  const waitForPriorDeploys = createWaitForPriorDeploysInitContainer(options.namespace, deployable.name, jobName);
 
   const helmContainer = await createHelmContainer(
     repository?.fullName || 'no-repo',
@@ -209,6 +254,7 @@ export async function generateHelmManifest(
     gitUsername: GIT_USERNAME,
     gitToken,
     cloneScript,
+    initContainers: [waitForPriorDeploys],
     containers: [helmContainer],
     volumes: volumeConfig.volumes,
     deployMetadata,

src/server/lib/nativeHelm/utils.ts

@@ -217,56 +217,7 @@ export function generateHelmInstallScript(
     postRenderer
   );
 
-  let script = [
-    'set -e',
-    `echo "Starting helm deployment for ${releaseName}"`,
-    '',
-    'apk add --no-cache -q jq',
-    '',
-    'KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)',
-    'KUBE_API="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"',
-    'KUBE_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt',
-    'SERVICE_NAME="${LC_SERVICE_NAME}"',
-    'MY_JOB_NAME="${LC_JOB_NAME}"',
-    'WAIT_TIMEOUT=900',
-    'POLL_INTERVAL=10',
-    '',
-    'echo "Checking for prior deploy jobs for service=${SERVICE_NAME}"',
-    '',
-    'my_created=$(wget -qO- --ca-certificate "$KUBE_CA" \\',
-    '  --header "Authorization: Bearer $KUBE_TOKEN" \\',
-    `  "\${KUBE_API}/apis/batch/v1/namespaces/${namespace}/jobs/\${MY_JOB_NAME}" \\`,
-    "  | jq -r '.metadata.creationTimestamp')",
-    '',
-    'wait_start=$(date +%s)',
-    'while true; do',
-    '  elapsed=$(( $(date +%s) - wait_start ))',
-    '  if [ "$elapsed" -ge "$WAIT_TIMEOUT" ]; then',
-    '    echo "ERROR: Timed out after ${WAIT_TIMEOUT}s waiting for prior deploy jobs to complete"',
-    '    exit 1',
-    '  fi',
-    '',
-    '  blocking_jobs=$(wget -qO- --ca-certificate "$KUBE_CA" \\',
-    '    --header "Authorization: Bearer $KUBE_TOKEN" \\',
-    `    "\${KUBE_API}/apis/batch/v1/namespaces/${namespace}/jobs?labelSelector=service=\${SERVICE_NAME},app.kubernetes.io/name=native-helm" \\`,
-    '    | jq -r --arg my_name "$MY_JOB_NAME" --arg my_ts "$my_created" \'',
-    '      [.items[] |',
-    '        select(.metadata.name != $my_name) |',
-    '        select(.metadata.creationTimestamp < $my_ts or',
-    '              (.metadata.creationTimestamp == $my_ts and .metadata.name < $my_name)) |',
-    '        select(.status.active > 0)',
-    '      | .metadata.name] | join(" ")\')',
-    '',
-    '  if [ -z "$blocking_jobs" ]; then',
-    '    echo "No prior deploy jobs in progress, proceeding"',
-    '    break',
-    '  fi',
-    '',
-    '  echo "Waiting for prior deploy jobs to complete: $blocking_jobs (${elapsed}s/${WAIT_TIMEOUT}s)"',
-    '  sleep $POLL_INTERVAL',
-    'done',
-    '',
-  ].join('\n');
+  let script = ['set -e', `echo "Starting helm deployment for ${releaseName}"`, ''].join('\n');
 
   if (repoName !== 'no-repo' && repoName.includes('/')) {
     script += `cd /workspace