diff --git a/src/server/lib/kubernetes/jobFactory.ts b/src/server/lib/kubernetes/jobFactory.ts index 2ea0c95..f7f124b 100644 --- a/src/server/lib/kubernetes/jobFactory.ts +++ b/src/server/lib/kubernetes/jobFactory.ts @@ -180,6 +180,7 @@ export interface HelmJobConfig { }; includeGitClone?: boolean; registryAuth?: RegistryAuthConfig; + initContainers?: any[]; } export function createHelmJob(config: HelmJobConfig): V1Job { @@ -221,6 +222,10 @@ export function createHelmJob(config: HelmJobConfig): V1Job { initContainers.push(createRegistryAuthInitContainer(config.registryAuth)); } + if (config.initContainers?.length) { + initContainers.push(...config.initContainers); + } + const containers = config.containers.map((container) => ({ ...container, resources: container.resources || { diff --git a/src/server/lib/nativeHelm/__tests__/helm.test.ts b/src/server/lib/nativeHelm/__tests__/helm.test.ts index 0341e15..bbb19b4 100644 --- a/src/server/lib/nativeHelm/__tests__/helm.test.ts +++ b/src/server/lib/nativeHelm/__tests__/helm.test.ts @@ -16,6 +16,7 @@ import { shouldUseNativeHelm, createHelmContainer, nativeHelmDeploy, generateHelmManifest } from '../helm'; import * as nativeHelmUtils from '../utils'; +import yaml from 'js-yaml'; import { determineChartType, constructHelmCommand, @@ -1537,6 +1538,9 @@ services: deployableId: 99, deployable: { name: 'sample-service', + repository: { + fullName: 'example-org/example-repo', + }, helm: helmConfig, }, build: { @@ -1633,7 +1637,7 @@ services: 'sample-chart': { chart: { name: 'sample-chart', - repoUrl: 'https://charts.example.com', + repoUrl: 'oci://123456789012.dkr.ecr.us-west-2.amazonaws.com/sample-chart', version: '2.5.1', valueFiles: ['global-values.yaml'], }, @@ -1695,13 +1699,21 @@ services: } as any; const manifest = await generateHelmManifest(deploy, 'test-job', { namespace: 'testns' }); + const parsed = yaml.load(manifest) as any; + const initContainerNames = parsed.spec.template.spec.initContainers.map((container: any) => container.name); + const helmContainer = parsed.spec.template.spec.containers.find( + (container: any) => container.name === 'helm-deploy' + ); expect(manifest).toContain('image: registry.example.com/service-helm-runner:2.0.0'); - expect(manifest).toContain("--post-renderer '/opt/bin/post-renderer'"); - expect(manifest).toContain("--post-renderer-args '--service-override'"); - expect(manifest).not.toContain("--post-renderer-args '--global'"); expect(manifest).toContain('-f service-values.yaml'); expect(manifest).not.toContain('-f global-values.yaml'); + expect(helmContainer.args[0]).toContain("--post-renderer '/opt/bin/post-renderer'"); + expect(helmContainer.args[0]).toContain("--post-renderer-args '--service-override'"); + expect(helmContainer.args[0]).not.toContain("--post-renderer-args '--global'"); + expect(initContainerNames[initContainerNames.length - 1]).toBe('wait-for-prior-deploys'); + expect(initContainerNames).toContain('ecr-auth'); + expect(initContainerNames.indexOf('ecr-auth')).toBeLessThan(initContainerNames.indexOf('wait-for-prior-deploys')); }); }); }); diff --git a/src/server/lib/nativeHelm/constants.ts b/src/server/lib/nativeHelm/constants.ts index a616b63..76f3a24 100644 --- a/src/server/lib/nativeHelm/constants.ts +++ b/src/server/lib/nativeHelm/constants.ts @@ -19,6 +19,8 @@ export const HELM_JOB_TIMEOUT_SECONDS = HELM_TIMEOUT_MINUTES * 60; export const STATIC_ENV_JOB_TTL_SECONDS = 86400; // 24 hours export const DEFAULT_HELM_VERSION = '3.12.0'; export const HELM_IMAGE_PREFIX = 'alpine/helm'; +export const HELM_WAIT_IMAGE = 'bitnamilegacy/kubectl:1.30'; +export const HELM_WAIT_TIMEOUT_SECONDS = 900; export const REPO_MAPPINGS = { bitnami: 'https://charts.bitnami.com/bitnami', diff --git a/src/server/lib/nativeHelm/helm.ts b/src/server/lib/nativeHelm/helm.ts index c0bba1f..dc5aed9 100644 --- a/src/server/lib/nativeHelm/helm.ts +++ b/src/server/lib/nativeHelm/helm.ts @@ -44,7 +44,7 @@ import { validateHelmConfiguration, } from './utils'; import { detectRegistryAuth, RegistryAuthConfig } from './registryAuth'; -import { HELM_IMAGE_PREFIX } from './constants'; +import { HELM_IMAGE_PREFIX, HELM_WAIT_IMAGE, HELM_WAIT_TIMEOUT_SECONDS } from './constants'; import { buildDeployJobName } from 'server/lib/kubernetes/jobNames'; import { createCloneScript, @@ -127,6 +127,50 @@ export async function createHelmContainer( }; } +export function createWaitForPriorDeploysInitContainer(namespace: string, serviceName: string, jobName: string): any { + const script = [ + 'set -euo pipefail', + `echo "Checking for prior deploy jobs for service=${serviceName}"`, + `MY_CREATED=$(kubectl get job ${jobName} -n ${namespace} -o jsonpath='{.metadata.creationTimestamp}')`, + `WAIT_TIMEOUT=${HELM_WAIT_TIMEOUT_SECONDS}`, + 'POLL_INTERVAL=10', + 'wait_start=$(date +%s)', + 'while true; do', + ' elapsed=$(( $(date +%s) - wait_start ))', + ' if [ "$elapsed" -ge "$WAIT_TIMEOUT" ]; then', + ' echo "ERROR: Timed out after ${WAIT_TIMEOUT}s waiting for prior deploy jobs to complete"', + ' exit 1', + ' fi', + '', + ` blocking_jobs=$(kubectl get jobs -n ${namespace} -l "service=${serviceName},app.kubernetes.io/name=native-helm" -o jsonpath='{range .items[*]}{.metadata.name}{"\\t"}{.metadata.creationTimestamp}{"\\t"}{.status.active}{"\\n"}{end}' | awk -v my_name="${jobName}" -v my_ts="$MY_CREATED" '`, + ' BEGIN { result = "" }', + ' $1 != my_name && (($2 < my_ts) || ($2 == my_ts && $1 < my_name)) && ($3 + 0) > 0 {', + ' result = result (result ? " " : "") $1', + ' }', + ` END { print result }')`, + '', + ' if [ -z "$blocking_jobs" ]; then', + ' echo "No prior deploy jobs in progress, proceeding"', + ' break', + ' fi', + '', + ' echo "Waiting for prior deploy jobs to complete: $blocking_jobs (${elapsed}s/${WAIT_TIMEOUT}s)"', + ' sleep $POLL_INTERVAL', + 'done', + ].join('\n'); + + return { + name: 'wait-for-prior-deploys', + image: HELM_WAIT_IMAGE, + command: ['/bin/bash', '-c'], + args: [script], + resources: { + requests: { cpu: '100m', memory: '128Mi' }, + limits: { cpu: '500m', memory: '512Mi' }, + }, + }; +} + export async function generateHelmManifest( deploy: Deploy, jobName: string, @@ -161,6 +205,7 @@ export async function generateHelmManifest( const registryAuth = detectRegistryAuth(chartRepoUrl); const helmImage = mergedHelmConfig.nativeHelm?.image; const postRenderer = mergedHelmConfig.nativeHelm?.postRenderer; + const waitForPriorDeploys = createWaitForPriorDeploysInitContainer(options.namespace, deployable.name, jobName); const helmContainer = await createHelmContainer( repository?.fullName || 'no-repo', @@ -209,6 +254,7 @@ export async function generateHelmManifest( gitUsername: GIT_USERNAME, gitToken, cloneScript, + initContainers: [waitForPriorDeploys], containers: [helmContainer], volumes: volumeConfig.volumes, deployMetadata, diff --git a/src/server/lib/nativeHelm/utils.ts b/src/server/lib/nativeHelm/utils.ts index 90b7b5d..26e7185 100644 --- a/src/server/lib/nativeHelm/utils.ts +++ b/src/server/lib/nativeHelm/utils.ts @@ -217,56 +217,7 @@ export function generateHelmInstallScript( postRenderer ); - let script = [ - 'set -e', - `echo "Starting helm deployment for ${releaseName}"`, - '', - 'apk add --no-cache -q jq', - '', - 'KUBE_TOKEN=$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)', - 'KUBE_API="https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}"', - 'KUBE_CA=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt', - 'SERVICE_NAME="${LC_SERVICE_NAME}"', - 'MY_JOB_NAME="${LC_JOB_NAME}"', - 'WAIT_TIMEOUT=900', - 'POLL_INTERVAL=10', - '', - 'echo "Checking for prior deploy jobs for service=${SERVICE_NAME}"', - '', - 'my_created=$(wget -qO- --ca-certificate "$KUBE_CA" \\', - ' --header "Authorization: Bearer $KUBE_TOKEN" \\', - ` "\${KUBE_API}/apis/batch/v1/namespaces/${namespace}/jobs/\${MY_JOB_NAME}" \\`, - " | jq -r '.metadata.creationTimestamp')", - '', - 'wait_start=$(date +%s)', - 'while true; do', - ' elapsed=$(( $(date +%s) - wait_start ))', - ' if [ "$elapsed" -ge "$WAIT_TIMEOUT" ]; then', - ' echo "ERROR: Timed out after ${WAIT_TIMEOUT}s waiting for prior deploy jobs to complete"', - ' exit 1', - ' fi', - '', - ' blocking_jobs=$(wget -qO- --ca-certificate "$KUBE_CA" \\', - ' --header "Authorization: Bearer $KUBE_TOKEN" \\', - ` "\${KUBE_API}/apis/batch/v1/namespaces/${namespace}/jobs?labelSelector=service=\${SERVICE_NAME},app.kubernetes.io/name=native-helm" \\`, - ' | jq -r --arg my_name "$MY_JOB_NAME" --arg my_ts "$my_created" \'', - ' [.items[] |', - ' select(.metadata.name != $my_name) |', - ' select(.metadata.creationTimestamp < $my_ts or', - ' (.metadata.creationTimestamp == $my_ts and .metadata.name < $my_name)) |', - ' select(.status.active > 0)', - ' | .metadata.name] | join(" ")\')', - '', - ' if [ -z "$blocking_jobs" ]; then', - ' echo "No prior deploy jobs in progress, proceeding"', - ' break', - ' fi', - '', - ' echo "Waiting for prior deploy jobs to complete: $blocking_jobs (${elapsed}s/${WAIT_TIMEOUT}s)"', - ' sleep $POLL_INTERVAL', - 'done', - '', - ].join('\n'); + let script = ['set -e', `echo "Starting helm deployment for ${releaseName}"`, ''].join('\n'); if (repoName !== 'no-repo' && repoName.includes('/')) { script += `cd /workspace