CUMULUS-4883: Add script to build Iceberg API docker image and push it to ECR as part of the build process.

CHANGELOG.md

@@ -15,6 +15,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Added
 
+- **CUMULUS-4883**
+  - Add script to build Iceberg API docker image and push it to ECR as part of the build process
 - **CUMULUS-4705**
   - Add Fargate task to cleanup old Iceberg table snapshots on a schedule
 - **CUMULUS-4711**

bamboo/bootstrap-tf-deployment.sh

@@ -53,7 +53,7 @@ echo "Deploying Cumulus data-persistence module to $DEPLOYMENT"
   -var "subnet_ids=[\"$AWS_SUBNET\"]" \
   -var "vpc_id=$VPC_ID" \
   -var "rds_admin_access_secret_arn=$RDS_ADMIN_ACCESS_SECRET_ARN" \
-  -var "rds_security_group=$RDS_SECURITY_GROUP"\
+  -var "rds_security_group=$RDS_SECURITY_GROUP" \
   -var "permissions_boundary_arn=arn:aws:iam::$AWS_ACCOUNT_ID:policy/$ROLE_BOUNDARY"
 
 cd ../cumulus-tf
@@ -71,6 +71,12 @@ echo "terraform {
 ../terraform init \
   -input=false
 
+set_iceberg_image_version
+
+DEPLOY_ICEBERG_API="${DEPLOY_ICEBERG_API:-false}"
+echo "Deploy Iceberg API: ${DEPLOY_ICEBERG_API}"
+echo "Using Iceberg API image version ${ICEBERG_IMAGE_VERSION}"
+
 # Deploy cumulus-tf via terraform
 echo "Deploying Cumulus example to $DEPLOYMENT"
 ../terraform apply \
@@ -101,3 +107,5 @@ echo "Deploying Cumulus example to $DEPLOYMENT"
   -var "metrics_es_host=$METRICS_ES_HOST" \
   -var "metrics_es_username=$METRICS_ES_USER" \
   -var "metrics_es_password=$METRICS_ES_PASS" \
+  -var "cumulus_iceberg_api_image_version=$ICEBERG_IMAGE_VERSION" \
+  -var "deploy_iceberg_api=$DEPLOY_ICEBERG_API"

bamboo/deploy-iceberg-api-image.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+set -ex
+
+. ./bamboo/use-working-directory.sh
+. ./bamboo/set-bamboo-env-variables.sh
+
+if [[ "$DEPLOY_ICEBERG_API" != "true" ]]; then
+  echo "Skipping deploy Iceberg API Image step (DEPLOY_ICEBERG_API=$DEPLOY_ICEBERG_API)" >&2
+  exit 0
+fi
+echo "***Deploying Iceberg API image"
+
+if ! command -v docker >/dev/null 2>&1; then
+  apt-get update && apt-get install -y docker.io
+fi
+
+set_iceberg_image_version
+
+image="cumulus-iceberg-api"
+registry_host="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com"
+aws ecr get-login-password --region "${AWS_REGION}" | docker login --username AWS --password-stdin "${registry_host}"
+# Create ECR repo if it doesn't exist
+if ! aws ecr describe-repositories --repository-names "${image}" >/dev/null 2>&1; then
+  if aws ecr describe-repositories --repository-names "${image}" 2>&1 | grep -q RepositoryNotFoundException; then
+    aws ecr create-repository --repository-name "${image}" >/dev/null
+  else
+    echo "Error: unable to verify ECR repository ${image}." >&2
+    exit 1
+  fi
+fi
+
+IMAGE_NAME="${registry_host}/${image}:${ICEBERG_IMAGE_VERSION}"
+
+echo "Building Iceberg API image with name=${IMAGE_NAME}"
+docker build --platform linux/arm64 -f packages/api/app/Dockerfile -t "$IMAGE_NAME" .
+
+echo "Publishing Docker image to ECR with name=${IMAGE_NAME}"
+docker push "$IMAGE_NAME"

bamboo/set-bamboo-env-variables.sh

@@ -58,6 +58,7 @@ declare -a param_list=(
   "bamboo_USE_TERRAFORM_ZIPS"
   "bamboo_VERSION_FLAG"
   "bamboo_CUMULUS_BASE_IMAGE"
+  "bamboo_DEPLOY_ICEBERG_API"
 )
 
 ## Strip 'bamboo_SECRET_' from secret keys
@@ -202,3 +203,22 @@ else
 fi
 export CUMULUS_UNIT_TEST_DATA=/tmp/cumulus_unit_test_data
 export TS_BUILD_CACHE_FILE=ts-build-cache.tgz
+
+## Function to set ICEBERG_IMAGE_VERSION based on PUBLISH_FLAG
+set_iceberg_image_version() {
+  echo "***Bamboo plan revision: $bamboo_plan_revision"
+  if [[ $PUBLISH_FLAG == true ]]; then
+    ICEBERG_IMAGE_VERSION=$(jq --raw-output .version lerna.json)
+  else
+    ICEBERG_IMAGE_VERSION=$(echo $bamboo_plan_revision | cut -c1-7)
+  fi
+
+  echo "***ICEBERG_IMAGE_VERSION: $ICEBERG_IMAGE_VERSION"
+
+  if [[ -z $ICEBERG_IMAGE_VERSION ]]; then
+    echo "Error: ICEBERG_IMAGE_VERSION is not set (PUBLISH_FLAG=${PUBLISH_FLAG}). Expected bamboo_plan_revision." >&2
+    exit 1
+  fi
+
+  export ICEBERG_IMAGE_VERSION
+}

example/deployments/cumulus/sandbox.tfvars

@@ -56,3 +56,6 @@ saml_launchpad_metadata_url = "https://auth.launchpad-sbx.nasa.gov/unauth/metada
 thin_egress_jwt_secret_name = "cumulus_sandbox_jwt_tea_secret"
 
 orca_default_bucket = "cumulus-test-sandbox-orca-glacier"
+
+iceberg_s3_bucket = "sandbox-ci-iceberg"
+iceberg_namespace = "sandbox_ci"

example/deployments/cumulus/yliu10-ci-tf.tfvars

@@ -1,5 +1,3 @@
 prefix = "yliu10-ci-tf"
 
 deploy_iceberg_api = true
-iceberg_s3_bucket = "yliu-sandbox-test-iceberg"
-iceberg_namespace = "yliu_test"

tf-modules/iceberg_api/main.tf

@@ -96,9 +96,6 @@ resource "aws_ecs_service" "iceberg_api" {
   }
 
   force_new_deployment = true
-  triggers = {
-    redeployment = sha256(jsonencode(aws_ecs_task_definition.iceberg_api))
-  }
 }
 
 resource "aws_lb" "iceberg_api" {