From 84960b23a847fe068f1bca62afed0cea21e962c2 Mon Sep 17 00:00:00 2001
From: Felix Delattre <felix@developmentseed.org>
Date: Wed, 10 Sep 2025 09:20:33 +0200
Subject: [PATCH 1/4] Reworked deploy, test and ingest scripts.

---
 Makefile                  |  28 +-
 scripts/README.md         |  78 ++++
 scripts/deploy.sh         | 234 ++++++++++++
 scripts/helm-template.sh  |  25 --
 scripts/ingest.sh         | 105 ++++--
 scripts/lib/README.md     |  61 ++++
 scripts/lib/common.sh     | 255 +++++++++++++
 scripts/release.sh        |  19 -
 scripts/rm-ci-releases.sh |  13 -
 scripts/test.sh           | 728 ++++++++++++++++++++++++++++++++++++++
 10 files changed, 1440 insertions(+), 106 deletions(-)
 create mode 100644 scripts/README.md
 create mode 100755 scripts/deploy.sh
 delete mode 100755 scripts/helm-template.sh
 create mode 100644 scripts/lib/README.md
 create mode 100755 scripts/lib/common.sh
 delete mode 100644 scripts/release.sh
 delete mode 100755 scripts/rm-ci-releases.sh
 create mode 100755 scripts/test.sh

diff --git a/Makefile b/Makefile
index c069f115..1d91c825 100755
--- a/Makefile
+++ b/Makefile
@@ -5,21 +5,15 @@ HELM_REPO_URL=https://devseed.com/eoapi-k8s/
 HELM_CHART_NAME=eoapi/eoapi
 PGO_CHART_VERSION=5.7.4
 
-.PHONY: all deploy minikube ingest help
+.PHONY: all deploy minikube ingest test-integration tests help
 
 # Default target
 all: deploy
 
 deploy:
-	@echo "Installing dependencies."
-	@command -v helm >/dev/null 2>&1 || { echo "helm is required but not installed"; exit 1; }
-	helm upgrade --install --set disable_check_for_upgrades=true pgo oci://registry.developers.crunchydata.com/crunchydata/pgo --version $(PGO_CHART_VERSION)
-	@echo "Adding eoAPI helm repository."
-	@helm repo add eoapi $(HELM_REPO_URL)
-	@echo "Installing eoAPI helm chart."
-	@cd ./charts && \
-	helm dependency build ./eoapi && \
-	helm upgrade --install --namespace eoapi --create-namespace --set gitSha=$$(git rev-parse HEAD | cut -c1-10) eoapi ./eoapi
+	@echo "Deploying eoAPI."
+	@command -v bash >/dev/null 2>&1 || { echo "bash is required but not installed"; exit 1; }
+	@./scripts/deploy.sh
 
 minikube:
 	@echo "Starting minikube."
@@ -37,13 +31,21 @@ ingest:
 	@./scripts/ingest.sh || { echo "Ingestion failed."; exit 1; }
 
 tests:
-	@echo "Running tests."
+	@echo "Running Helm unit tests..."
 	@command -v helm >/dev/null 2>&1 || { echo "helm is required but not installed"; exit 1; }
-	@helm unittest charts/eoapi -f 'tests/*.yaml' -v charts/eoapi/test-helm-values.yaml
+	@./scripts/deploy.sh setup
+	@./scripts/test.sh helm
+
+integration:
+	@echo "Running integration tests against Kubernetes cluster..."
+	@command -v bash >/dev/null 2>&1 || { echo "bash is required but not installed"; exit 1; }
+	@./scripts/test.sh integration
 
 help:
 	@echo "Makefile commands:"
-	@echo "  make deploy         -  Install eoAPI on a cluster kubectl is connected to."
+	@echo "  make deploy         -  Deploy eoAPI with on connected Kubernetes cluster."
 	@echo "  make minikube       -  Install eoAPI on minikube."
 	@echo "  make ingest         -  Ingest STAC collections and items into the database."
+	@echo "  make integration    -  Run integration tests on connected Kubernetes cluster."
+	@echo "  make tests          -  Run lint + unit tests."
 	@echo "  make help           -  Show this help message."
diff --git a/scripts/README.md b/scripts/README.md
new file mode 100644
index 00000000..c93b9674
--- /dev/null
+++ b/scripts/README.md
@@ -0,0 +1,78 @@
+# eoAPI Kubernetes Scripts
+
+Automation scripts for deploying, testing, and managing eoAPI on Kubernetes.
+
+## Scripts Overview
+
+| Script | Purpose | Usage |
+|--------|---------|-------|
+| **`deploy.sh`** | Deploy eoAPI to Kubernetes | `./deploy.sh [deploy\|setup\|cleanup] [--ci]` |
+| **`ingest.sh`** | Ingest STAC data into deployed eoAPI | `./ingest.sh [collections.json] [items.json]` |
+| **`test.sh`** | Run Helm and integration tests | `./test.sh [helm\|integration\|all] [--debug]` |
+| **`lib/`** | Shared utility functions | See [lib/README.md](lib/README.md) |
+
+## Quick Start
+
+```bash
+# Deploy eoAPI
+./scripts/deploy.sh
+
+# Ingest sample data
+./scripts/ingest.sh collections.json items.json
+
+# Run tests
+./scripts/test.sh
+```
+
+## Prerequisites
+
+- **kubectl** - Kubernetes CLI configured for your cluster
+- **helm** - Helm package manager v3+
+- **python3** - For data ingestion and testing
+- **jq** - JSON processor (for advanced features)
+
+## Environment Variables (Optional)
+
+Most settings are auto-detected. Override only when needed:
+
+```bash
+# Deployment customization
+export PGO_VERSION=5.7.4           # PostgreSQL operator version
+export TIMEOUT=15m                  # Deployment timeout
+
+# Override auto-detection (usually not needed)
+export NAMESPACE=my-eoapi           # Target namespace  
+export RELEASE_NAME=my-release      # Helm release name
+
+# Testing endpoints (auto-detected by test.sh)
+export STAC_ENDPOINT=http://...     # Override STAC API endpoint
+export RASTER_ENDPOINT=http://...   # Override Raster API endpoint  
+export VECTOR_ENDPOINT=http://...   # Override Vector API endpoint
+```
+
+## Common Examples
+
+**Deploy with custom namespace:**
+```bash
+NAMESPACE=my-eoapi ./scripts/deploy.sh
+```
+
+**Setup dependencies only:**
+```bash
+./scripts/deploy.sh setup
+```
+
+**Run tests with debug output:**
+```bash
+./scripts/test.sh all --debug
+```
+
+**Cleanup deployment:**
+```bash
+./scripts/deploy.sh cleanup
+```
+
+**CI mode deployment:**
+```bash
+./scripts/deploy.sh --ci
+```
diff --git a/scripts/deploy.sh b/scripts/deploy.sh
new file mode 100755
index 00000000..d8a7bc7f
--- /dev/null
+++ b/scripts/deploy.sh
@@ -0,0 +1,234 @@
+#!/bin/bash
+
+# eoAPI Deployment Script
+
+# Source shared utilities
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+source "$SCRIPT_DIR/lib/common.sh"
+
+# Default values
+PGO_VERSION="${PGO_VERSION:-5.7.4}"
+RELEASE_NAME="${RELEASE_NAME:-eoapi}"
+NAMESPACE="${NAMESPACE:-eoapi}"
+TIMEOUT="${TIMEOUT:-10m}"
+CI_MODE=false
+COMMAND=""
+
+# Auto-detect CI environment
+CI_MODE=$(is_ci_environment && echo true || echo false)
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        deploy|setup|cleanup)
+            COMMAND="$1"; shift ;;
+        --ci) CI_MODE=true; shift ;;
+        --help|-h)
+            echo "eoAPI Deployment Script"
+            echo "Usage: $(basename "$0") [COMMAND] [OPTIONS]"
+            echo ""
+            echo "Commands:"
+            echo "  deploy     Deploy eoAPI (includes setup) [default]"
+            echo "  setup      Setup Helm dependencies only"
+            echo "  cleanup    Cleanup deployment resources"
+            echo ""
+            echo "Options:"
+            echo "  --ci       Enable CI mode"
+            echo "  --help     Show this help message"
+            echo ""
+            echo "Environment variables:"
+            echo "  PGO_VERSION    PostgreSQL Operator version (default: 5.7.4)"
+            echo "  RELEASE_NAME   Helm release name (default: eoapi)"
+            echo "  NAMESPACE      Kubernetes namespace (default: eoapi)"
+            echo "  TIMEOUT        Helm install timeout (default: 10m)"
+            exit 0 ;;
+        *) log_error "Unknown option: $1"; exit 1 ;;
+    esac
+done
+
+# Default to deploy if no command specified
+if [ -z "$COMMAND" ]; then
+    COMMAND="deploy"
+fi
+
+log_info "Starting eoAPI $COMMAND$([ "$CI_MODE" = true ] && echo " (CI MODE)" || echo "")..."
+log_info "Release: $RELEASE_NAME | Namespace: $NAMESPACE | PGO Version: $PGO_VERSION"
+
+# Run pre-flight checks (skip for setup-only mode)
+if [ "$COMMAND" != "setup" ]; then
+    preflight_deploy || exit 1
+fi
+
+# Install PostgreSQL operator
+install_pgo() {
+    log_info "Installing PostgreSQL Operator..."
+    if helm list -A -q | grep -q "^pgo$"; then
+        log_info "PGO already installed, upgrading..."
+        helm upgrade pgo oci://registry.developers.crunchydata.com/crunchydata/pgo \
+            --version "$PGO_VERSION" --set disable_check_for_upgrades=true
+    else
+        helm install pgo oci://registry.developers.crunchydata.com/crunchydata/pgo \
+            --version "$PGO_VERSION" --set disable_check_for_upgrades=true
+    fi
+
+    # Wait for PostgreSQL operator
+    log_info "Waiting for PostgreSQL Operator to be ready..."
+    if ! kubectl wait --for=condition=Available deployment/pgo --timeout=300s; then
+        log_error "PostgreSQL Operator failed to become ready"
+        kubectl get pods -l postgres-operator.crunchydata.com/control-plane=postgres-operator
+        exit 1
+    fi
+    kubectl get pods -l postgres-operator.crunchydata.com/control-plane=postgres-operator
+}
+
+# Integrated Helm dependency setup
+setup_helm_dependencies() {
+    log_info "Setting up Helm dependencies..."
+
+    # Add repositories from Chart.yaml files
+    for chart in charts/*/; do
+        if [ -f "$chart/Chart.yaml" ]; then
+            log_info "Processing $chart..."
+
+            # Extract unique repository URLs
+            if grep -q "repository:" "$chart/Chart.yaml" 2>/dev/null; then
+                grep "repository:" "$chart/Chart.yaml" 2>/dev/null | \
+                sed "s/.*repository: *//" | \
+                grep -v "file://" | \
+                sort -u | \
+                while read -r repo; do
+                    if [ -n "$repo" ]; then
+                        repo_name=$(echo "$repo" | sed "s|https://||" | sed "s|/.*||" | sed "s/\./-/g")
+                        log_info "Adding repository $repo_name -> $repo"
+                        helm repo add "$repo_name" "$repo" 2>/dev/null || true
+                    fi
+                done
+            fi
+        fi
+    done
+
+    # Update repositories
+    log_info "Updating helm repositories..."
+    helm repo update
+
+    # Build dependencies
+    for chart in charts/*/; do
+        if [ -f "$chart/Chart.yaml" ]; then
+            log_info "Building dependencies for $chart..."
+            (
+                cd "$chart" || exit
+                helm dependency build
+            )
+        fi
+    done
+
+    log_info "✅ Helm dependency setup complete"
+}
+
+# Deploy eoAPI function
+deploy_eoapi() {
+    log_info "Deploying eoAPI..."
+    cd charts || exit
+
+    # Build Helm command
+    HELM_CMD="helm upgrade --install $RELEASE_NAME ./eoapi"
+    HELM_CMD="$HELM_CMD --namespace $NAMESPACE --create-namespace"
+    HELM_CMD="$HELM_CMD --timeout=$TIMEOUT"
+
+    # Add values files
+    if [ -f "./eoapi/values.yaml" ]; then
+        HELM_CMD="$HELM_CMD -f ./eoapi/values.yaml"
+    fi
+
+    # CI-specific configuration
+    if [ "$CI_MODE" = true ] && [ -f "./eoapi/test-k3s-unittest-values.yaml" ]; then
+        log_info "Using CI test configuration..."
+        HELM_CMD="$HELM_CMD -f ./eoapi/test-k3s-unittest-values.yaml"
+    fi
+
+    # Set git SHA if available
+    if [ -n "$GITHUB_SHA" ]; then
+        HELM_CMD="$HELM_CMD --set gitSha=$GITHUB_SHA"
+    elif [ -n "$(git rev-parse HEAD 2>/dev/null)" ]; then
+        HELM_CMD="$HELM_CMD --set gitSha=$(git rev-parse HEAD | cut -c1-10)"
+    fi
+
+    # Execute deployment
+    log_info "Running: $HELM_CMD"
+    eval "$HELM_CMD"
+
+    cd .. || exit
+
+    # Verify deployment
+    log_info "Verifying deployment..."
+    kubectl get pods -n "$NAMESPACE" -o wide
+
+    log_info "eoAPI deployment completed successfully!"
+    log_info "Services available in namespace: $NAMESPACE"
+
+    if [ "$CI_MODE" != true ]; then
+        log_info "To run integration tests: make integration"
+        log_info "To check status: kubectl get pods -n $NAMESPACE"
+    fi
+}
+
+# Cleanup function
+cleanup_deployment() {
+    log_info "Cleaning up resources for release: $RELEASE_NAME"
+
+    # Validate namespace exists
+    if ! validate_namespace "$NAMESPACE"; then
+        log_warn "Namespace '$NAMESPACE' not found, skipping cleanup"
+        return 0
+    fi
+
+    # Function to safely delete resources
+    cleanup_resource() {
+        local resource_type="$1"
+        local resources
+
+        log_info "Cleaning up ${resource_type}..."
+        resources=$(kubectl get "$resource_type" -n "$NAMESPACE" --no-headers 2>/dev/null | grep "$RELEASE_NAME" | awk '{print $1}' || true)
+
+        if [ -n "$resources" ]; then
+            log_info "  Found ${resource_type}: $resources"
+            echo "$resources" | xargs -r kubectl delete "$resource_type" -n "$NAMESPACE"
+        else
+            log_info "  No ${resource_type} found for $RELEASE_NAME"
+        fi
+    }
+
+    # Clean up resources in order (dependencies first)
+    cleanup_resource "ingress"
+    cleanup_resource "service"
+    cleanup_resource "deployment"
+    cleanup_resource "job"
+    cleanup_resource "configmap"
+    cleanup_resource "secret"
+    cleanup_resource "pvc"
+
+    # Try helm uninstall as well (if it's a helm release)
+    log_info "Attempting helm uninstall..."
+    helm uninstall "$RELEASE_NAME" -n "$NAMESPACE" 2>/dev/null || log_warn "No helm release found for $RELEASE_NAME"
+
+    log_info "✅ Cleanup complete for release: $RELEASE_NAME"
+}
+
+# Execute based on command
+case $COMMAND in
+    setup)
+        setup_helm_dependencies
+        ;;
+    cleanup)
+        cleanup_deployment
+        ;;
+    deploy)
+        install_pgo
+        setup_helm_dependencies
+        deploy_eoapi
+        ;;
+    *)
+        log_error "Unknown command: $COMMAND"
+        exit 1
+        ;;
+esac
diff --git a/scripts/helm-template.sh b/scripts/helm-template.sh
deleted file mode 100755
index d499bf26..00000000
--- a/scripts/helm-template.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-export PGUSER=username
-export POSTGRES_USER=username
-export PGPASSWORD=password
-export POSTGRES_PASSWORD=password
-export GITSHA=$(git rev-parse HEAD | cut -c1-10)
-
-mkdir -p tmp
-MANIFEST=./tmp/eoapi-manfests.yaml
-
-echo "" > $MANIFEST
-MANIFESTS=$(find ./templates/* -name "*.yaml")
-while read MFILE; do
-  path_without_dot_prefix=$(echo "$MFILE" | sed 's/^\.\///g')
-  printf "[ RENDERING ]: %s\n" "$path_without_dot_prefix"
-  helm template . \
-    -s $path_without_dot_prefix \
-    --set gitSha=$GITSHA \
-    --set db.settings.secrets.PGUSER=$PGUSER \
-    --set db.settings.secrets.POSTGRES_USER=$POSTGRES_USER \
-    --set db.settings.secrets.PGPASSWORD=$PGPASSWORD \
-    --set db.settings.secrets.POSTGRES_PASSWORD=$POSTGRES_PASSWORD \
-    -f values.yaml >> $MANIFEST
-done < <(echo "$MANIFESTS")
-
diff --git a/scripts/ingest.sh b/scripts/ingest.sh
index ce4a200d..30da65df 100755
--- a/scripts/ingest.sh
+++ b/scripts/ingest.sh
@@ -1,5 +1,11 @@
 #!/bin/bash
 
+# eoAPI Data Ingestion Script
+
+# Source shared utilities
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+source "$SCRIPT_DIR/lib/common.sh"
+
 # Default files
 DEFAULT_COLLECTIONS_FILE="./collections.json"
 DEFAULT_ITEMS_FILE="./items.json"
@@ -11,69 +17,96 @@ if [ "$#" -eq 2 ]; then
 else
     EOAPI_COLLECTIONS_FILE="$DEFAULT_COLLECTIONS_FILE"
     EOAPI_ITEMS_FILE="$DEFAULT_ITEMS_FILE"
-    echo "No specific files provided. Using defaults:"
-    echo "  Collections file: $EOAPI_COLLECTIONS_FILE"
-    echo "  Items file: $EOAPI_ITEMS_FILE"
+    log_info "No specific files provided. Using defaults:"
+    log_info "  Collections file: $EOAPI_COLLECTIONS_FILE"
+    log_info "  Items file: $EOAPI_ITEMS_FILE"
+fi
+
+# Run pre-flight checks
+if ! preflight_ingest "$(detect_namespace)" "$EOAPI_COLLECTIONS_FILE" "$EOAPI_ITEMS_FILE"; then
+    exit 1
 fi
 
-# Define namespaces
-NAMESPACES=("default" "eoapi" "data-access")
+# Detect namespace and raster pod
+FOUND_NAMESPACE=$(detect_namespace)
+log_info "Using namespace: $FOUND_NAMESPACE"
+
+# Find raster pod using multiple patterns
 EOAPI_POD_RASTER=""
-FOUND_NAMESPACE=""
+PATTERNS=(
+    "app=raster-eoapi"
+    "app.kubernetes.io/name=raster"
+    "app.kubernetes.io/component=raster"
+)
 
-# Discover the pod name from both namespaces
-for NS in "${NAMESPACES[@]}"; do
-    EOAPI_POD_RASTER=$(kubectl get pods -n "$NS" -l app=raster-eoapi -o jsonpath="{.items[0].metadata.name}" 2>/dev/null)
+for pattern in "${PATTERNS[@]}"; do
+    EOAPI_POD_RASTER=$(kubectl get pods -n "$FOUND_NAMESPACE" -l "$pattern" -o jsonpath="{.items[0].metadata.name}" 2>/dev/null || echo "")
     if [ -n "$EOAPI_POD_RASTER" ]; then
-        FOUND_NAMESPACE="$NS"
-        echo "Found raster-eoapi pod: $EOAPI_POD_RASTER in namespace: $FOUND_NAMESPACE"
+        log_info "Found raster pod: $EOAPI_POD_RASTER (pattern: $pattern)"
         break
     fi
 done
 
 # Check if the pod was found
 if [ -z "$EOAPI_POD_RASTER" ]; then
-    echo "Could not determine raster-eoapi pod."
+    log_error "Could not find raster pod in namespace: $FOUND_NAMESPACE"
+    log_error "Available pods:"
+    kubectl get pods -n "$FOUND_NAMESPACE" -o name 2>/dev/null || true
+    exit 1
+fi
+
+# Validate pod is ready
+log_info "Validating pod readiness..."
+if ! kubectl wait --for=condition=Ready pod "$EOAPI_POD_RASTER" -n "$FOUND_NAMESPACE" --timeout=30s; then
+    log_error "Pod $EOAPI_POD_RASTER is not ready"
+    kubectl describe pod "$EOAPI_POD_RASTER" -n "$FOUND_NAMESPACE"
     exit 1
 fi
 
-# Check if input files exist
-for FILE in "$EOAPI_COLLECTIONS_FILE" "$EOAPI_ITEMS_FILE"; do
-    if [ ! -f "$FILE" ]; then
-        echo "File not found: $FILE. You may set them via the EOAPI_COLLECTIONS_FILE and EOAPI_ITEMS_FILE environment variables."
+# Check if pypgstac is already available (avoid unnecessary installations)
+log_info "Checking for pypgstac in pod..."
+if kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- python3 -c "import pypgstac" >/dev/null 2>&1; then
+    log_info "pypgstac already available"
+else
+    log_info "Installing pypgstac in pod $EOAPI_POD_RASTER..."
+    if ! kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c 'pip install pypgstac[psycopg]'; then
+        log_error "Failed to install pypgstac"
         exit 1
     fi
-done
+fi
+
+# Copy files to pod
+log_info "Copying files to pod..."
+log_info "  Collections: $EOAPI_COLLECTIONS_FILE"
+log_info "  Items: $EOAPI_ITEMS_FILE"
 
-# Install required packages
-echo "Installing required packages in pod $EOAPI_POD_RASTER in namespace $FOUND_NAMESPACE..."
-if ! kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c 'apt update -y && apt install python3 python3-pip -y && pip install pypgstac[psycopg]'; then
-    echo "Failed to install packages."
+if ! kubectl cp "$EOAPI_COLLECTIONS_FILE" "$FOUND_NAMESPACE/$EOAPI_POD_RASTER":/tmp/collections.json; then
+    log_error "Failed to copy collections file"
     exit 1
 fi
 
-# Copy files to pod
-echo "Copying files to pod..."
-echo "Using collections file: $EOAPI_COLLECTIONS_FILE"
-echo "Using items file: $EOAPI_ITEMS_FILE"
-kubectl cp "$EOAPI_COLLECTIONS_FILE" "$FOUND_NAMESPACE/$EOAPI_POD_RASTER":/tmp/collections.json
-kubectl cp "$EOAPI_ITEMS_FILE" "$FOUND_NAMESPACE/$EOAPI_POD_RASTER":/tmp/items.json
+if ! kubectl cp "$EOAPI_ITEMS_FILE" "$FOUND_NAMESPACE/$EOAPI_POD_RASTER":/tmp/items.json; then
+    log_error "Failed to copy items file"
+    exit 1
+fi
 
 # Load collections and items
-echo "Loading collections..."
-if ! kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c 'pypgstac load collections /tmp/collections.json --dsn "$PGADMIN_URI" --method insert_ignore'; then
-    echo "Failed to load collections."
+log_info "Loading collections..."
+if ! kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c "pypgstac load collections /tmp/collections.json --dsn \"\$PGADMIN_URI\" --method insert_ignore"; then
+    log_error "Failed to load collections"
     exit 1
 fi
+log_info "Collections loaded successfully"
 
-echo "Loading items..."
-if ! kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c 'pypgstac load items /tmp/items.json --dsn "$PGADMIN_URI" --method insert_ignore'; then
-    echo "Failed to load items."
+log_info "Loading items..."
+if ! kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c "pypgstac load items /tmp/items.json --dsn \"\$PGADMIN_URI\" --method insert_ignore"; then
+    log_error "Failed to load items"
     exit 1
 fi
+log_info "Items loaded successfully"
 
 # Clean temporary files
-echo "Cleaning temporary files..."
-kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c 'rm -f /tmp/collection.json /tmp/items.json'
+log_info "Cleaning temporary files..."
+kubectl exec -n "$FOUND_NAMESPACE" "$EOAPI_POD_RASTER" -- bash -c 'rm -f /tmp/collections.json /tmp/items.json' || log_warn "Failed to clean temporary files"
 
-echo "Ingestion complete."
+log_info "✅ Ingestion completed successfully"
diff --git a/scripts/lib/README.md b/scripts/lib/README.md
new file mode 100644
index 00000000..1fb2f612
--- /dev/null
+++ b/scripts/lib/README.md
@@ -0,0 +1,61 @@
+# eoAPI Scripts - Shared Utilities
+
+This directory contains shared utility functions used across eoAPI deployment, testing, and ingestion scripts.
+
+## Usage
+
+Source the common utilities in your scripts:
+
+```bash
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+source "$SCRIPT_DIR/lib/common.sh"
+```
+
+## Available Functions
+
+### Logging
+- `log_info "message"` - Info messages (green)
+- `log_warn "message"` - Warning messages (yellow) 
+- `log_error "message"` - Error messages (red)
+- `log_debug "message"` - Debug messages (blue)
+
+### Validation
+- `command_exists "tool"` - Check if command is available
+- `validate_tools tool1 tool2 ...` - Validate required tools exist
+- `validate_cluster` - Check Kubernetes cluster connectivity
+- `validate_namespace "namespace"` - Check if namespace exists
+- `validate_eoapi_deployment "namespace" "release"` - Validate eoAPI deployment
+
+### Detection
+- `is_ci_environment` - Returns true if running in CI
+- `detect_release_name ["namespace"]` - Auto-detect eoAPI release name
+- `detect_namespace` - Auto-detect eoAPI namespace
+
+### Utilities
+- `wait_for_pods "namespace" "selector" ["timeout"]` - Wait for pods to be ready
+
+### Pre-flight Checks
+- `preflight_deploy` - Validate deployment prerequisites
+- `preflight_ingest "namespace" "collections_file" "items_file"` - Validate ingestion prerequisites
+- `preflight_test "helm|integration"` - Validate test prerequisites
+
+## Error Handling
+
+All functions use proper error handling with `set -euo pipefail`. Scripts automatically exit on errors with descriptive messages.
+
+## Example
+
+```bash
+#!/bin/bash
+source "$(dirname "$0")/lib/common.sh"
+
+# Validate prerequisites
+preflight_deploy || exit 1
+
+# Use utilities
+NAMESPACE=$(detect_namespace)
+RELEASE=$(detect_release_name "$NAMESPACE")
+
+log_info "Deploying $RELEASE to $NAMESPACE"
+validate_eoapi_deployment "$NAMESPACE" "$RELEASE"
+```
diff --git a/scripts/lib/common.sh b/scripts/lib/common.sh
new file mode 100755
index 00000000..dcaf18d6
--- /dev/null
+++ b/scripts/lib/common.sh
@@ -0,0 +1,255 @@
+#!/bin/bash
+
+# eoAPI Scripts - Shared Utilities Library
+# Source this file in other scripts: source "$(dirname "$0")/lib/common.sh"
+
+set -euo pipefail
+
+# Colors
+readonly RED='\033[0;31m'
+readonly GREEN='\033[0;32m'
+readonly YELLOW='\033[1;33m'
+readonly BLUE='\033[0;34m'
+readonly NC='\033[0m'
+
+# Logging functions
+log_info() { echo -e "${GREEN}[INFO]${NC} $1" >&2; }
+log_warn() { echo -e "${YELLOW}[WARN]${NC} $1" >&2; }
+log_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }
+log_debug() { echo -e "${BLUE}[DEBUG]${NC} $1" >&2; }
+
+# Check if command exists
+command_exists() {
+    command -v "$1" >/dev/null 2>&1
+}
+
+# Validate required tools
+validate_tools() {
+    local tools=("$@")
+    local missing=()
+
+    for tool in "${tools[@]}"; do
+        if ! command_exists "$tool"; then
+            missing+=("$tool")
+        fi
+    done
+
+    if [ ${#missing[@]} -ne 0 ]; then
+        log_error "Missing required tools: ${missing[*]}"
+        return 1
+    fi
+
+    log_debug "All required tools available: ${tools[*]}"
+    return 0
+}
+
+# Check Kubernetes cluster connectivity
+validate_cluster() {
+    if ! kubectl cluster-info >/dev/null 2>&1; then
+        log_error "Cannot connect to Kubernetes cluster"
+        log_error "Ensure kubectl is configured and cluster is accessible"
+        return 1
+    fi
+
+    local context
+    context=$(kubectl config current-context 2>/dev/null || echo "unknown")
+    log_debug "Connected to cluster: $context"
+    return 0
+}
+
+# Detect CI environment
+is_ci_environment() {
+    [[ -n "${CI:-}" || -n "${GITHUB_ACTIONS:-}" || -n "${GITLAB_CI:-}" || -n "${JENKINS_URL:-}" ]]
+}
+
+# Validate namespace exists or can be created
+validate_namespace() {
+    local namespace="${1:-}"
+
+    if [ -z "$namespace" ]; then
+        log_error "Namespace not specified"
+        return 1
+    fi
+
+    if kubectl get namespace "$namespace" >/dev/null 2>&1; then
+        log_debug "Namespace '$namespace' exists"
+        return 0
+    fi
+
+    log_warn "Namespace '$namespace' does not exist"
+    return 1
+}
+
+# Auto-detect release name from deployed resources
+detect_release_name() {
+    local namespace="${1:-}"
+
+    # Try helm releases first
+    local release_name
+    release_name=$(helm list ${namespace:+-n "$namespace"} -o json 2>/dev/null | \
+                  jq -r '.[] | select(.name | contains("eoapi")) | .name' 2>/dev/null | head -1 || echo "")
+
+    # Fallback to pod labels
+    if [ -z "$release_name" ]; then
+        release_name=$(kubectl get pods ${namespace:+-n "$namespace"} \
+                      -l app.kubernetes.io/name=stac -o jsonpath='{.items[0].metadata.labels.app\.kubernetes\.io/instance}' \
+                      2>/dev/null || echo "eoapi")
+    fi
+
+    echo "${release_name:-eoapi}"
+}
+
+# Auto-detect namespace from deployed eoAPI resources
+detect_namespace() {
+    kubectl get pods --all-namespaces -l app.kubernetes.io/name=stac \
+        -o jsonpath='{.items[0].metadata.namespace}' 2>/dev/null || echo "eoapi"
+}
+
+# Wait for pods with label selector
+wait_for_pods() {
+    local namespace="$1"
+    local selector="$2"
+    local timeout="${3:-300s}"
+
+    log_info "Waiting for pods with selector: $selector"
+
+    if ! kubectl wait --for=condition=Ready pod -l "$selector" -n "$namespace" --timeout="$timeout" 2>/dev/null; then
+        log_error "Pods failed to become ready: $selector"
+        kubectl get pods -n "$namespace" -l "$selector" -o wide 2>/dev/null || true
+        return 1
+    fi
+
+    return 0
+}
+
+# Check if eoAPI is deployed
+validate_eoapi_deployment() {
+    local namespace="$1"
+    local release_name="$2"
+
+    log_info "Validating eoAPI deployment in namespace: $namespace"
+
+    local services=("stac" "raster" "vector")
+    local missing_services=()
+
+    for service in "${services[@]}"; do
+        local patterns=(
+            "app.kubernetes.io/instance=$release_name,app.kubernetes.io/name=$service"
+            "app=$release_name-$service"
+        )
+
+        local found=false
+        for pattern in "${patterns[@]}"; do
+            if kubectl get pods -n "$namespace" -l "$pattern" >/dev/null 2>&1; then
+                found=true
+                break
+            fi
+        done
+
+        if [ "$found" = false ]; then
+            missing_services+=("$service")
+        fi
+    done
+
+    if [ ${#missing_services[@]} -ne 0 ]; then
+        log_error "Missing eoAPI services: ${missing_services[*]}"
+        return 1
+    fi
+
+    log_info "eoAPI deployment validated successfully"
+    return 0
+}
+
+# Pre-flight checks for deployment
+preflight_deploy() {
+    log_info "Running pre-flight checks for deployment..."
+
+    validate_tools kubectl helm || return 1
+    validate_cluster || return 1
+
+    # Check Helm repositories are accessible
+    if ! helm repo list >/dev/null 2>&1; then
+        log_warn "No Helm repositories configured"
+    fi
+
+    log_info "✅ Pre-flight checks passed"
+    return 0
+}
+
+# Pre-flight checks for ingestion
+preflight_ingest() {
+    local namespace="$1"
+    local collections_file="$2"
+    local items_file="$3"
+
+    log_info "Running pre-flight checks for ingestion..."
+
+    validate_tools kubectl || return 1
+    validate_cluster || return 1
+    validate_namespace "$namespace" || return 1
+
+    # Check input files
+    for file in "$collections_file" "$items_file"; do
+        if [ ! -f "$file" ]; then
+            log_error "Input file not found: $file"
+            return 1
+        fi
+
+        if [ ! -s "$file" ]; then
+            log_error "Input file is empty: $file"
+            return 1
+        fi
+
+        # Basic JSON validation
+        if ! python3 -m json.tool "$file" >/dev/null 2>&1; then
+            log_error "Invalid JSON in file: $file"
+            return 1
+        fi
+    done
+
+    log_info "✅ Pre-flight checks passed"
+    return 0
+}
+
+# Pre-flight checks for testing
+preflight_test() {
+    local test_type="$1"
+
+    log_info "Running pre-flight checks for $test_type tests..."
+
+    case "$test_type" in
+        helm)
+            validate_tools helm || return 1
+            ;;
+        integration)
+            validate_tools kubectl python3 || return 1
+            validate_cluster || return 1
+            ;;
+        *)
+            log_error "Unknown test type: $test_type"
+            return 1
+            ;;
+    esac
+
+    log_info "✅ Pre-flight checks passed"
+    return 0
+}
+
+# Cleanup function for trapped errors
+cleanup_on_exit() {
+    local exit_code=$?
+    if [ $exit_code -ne 0 ]; then
+        log_error "Script failed with exit code: $exit_code"
+    fi
+}
+
+# Set up error handling
+trap cleanup_on_exit EXIT
+
+# Export functions for use in other scripts
+export -f log_info log_warn log_error log_debug
+export -f command_exists validate_tools validate_cluster
+export -f is_ci_environment validate_namespace
+export -f detect_release_name detect_namespace
+export -f wait_for_pods validate_eoapi_deployment
+export -f preflight_deploy preflight_ingest preflight_test
diff --git a/scripts/release.sh b/scripts/release.sh
deleted file mode 100644
index 9a4cb5cb..00000000
--- a/scripts/release.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-
-export RELEASE_NAME=eoapi
-export RELEASE_NS=eoapi
-export SUPPORT_RELEASE_NAME=eoapi-support
-export SUPPORT_RELEASE_NS=eoapi
-PROMETHEUS_SERVER="http://${SUPPORT_RELEASE_NAME}-prometheus-server.${SUPPORT_RELEASE_NS}.svc.cluster.local"
-
-helm upgrade --install \
-  -n $SUPPORT_RELEASE_NS --create-namespace $SUPPORT_RELEASE_NAME \
-  eoapi/eoapi-support --version 0.1.5 \
-  --set prometheus-adapter.prometheus.url=$PROMETHEUS_SERVER \
-  --set grafana.datasources.datasources\\.yaml.datasources[0].url=$PROMETHEUS_SERVER \
-  -f ./charts/eoapi-support/values.yaml
-
-helm upgrade --install \
-  -n $RELEASE_NS --create-namespace $RELEASE_NAME \
-  eoapi/eoapi --version 0.4.8 \
-  -f ./charts/eoapi/values.yaml
diff --git a/scripts/rm-ci-releases.sh b/scripts/rm-ci-releases.sh
deleted file mode 100755
index caee6eaf..00000000
--- a/scripts/rm-ci-releases.sh
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/bash
-# rm deploys (which will remove replicasets and pods)
-kubectl get deploy | grep -v NAME | grep $1 | cut -d' ' -f1 | xargs -I{} kubectl delete deploy/{}
-# rm svc
-kubectl get svc | grep -v NAME | grep $1 | cut -d' ' -f1 | xargs -I{} kubectl delete svc/{}
-# rm ingress
-kubectl get ingress | grep -v NAME | grep $1 | cut -d' ' -f1 | xargs -I{} kubectl delete ingress/{}
-# rm pvc
-kubectl get pvc | grep -v NAME | grep $1 | cut -d' ' -f1 | xargs -I{} kubectl delete pvc/{}
-# rm secrets
-kubectl get secret | grep -v NAME | grep $1 | cut -d' ' -f1 | xargs -I{} kubectl delete secret/{}
-# rm configmap
-kubectl get configmap | grep -v NAME | grep $1 | cut -d' ' -f1 | xargs -I{} kubectl delete configmap/{}
diff --git a/scripts/test.sh b/scripts/test.sh
new file mode 100755
index 00000000..70d72c79
--- /dev/null
+++ b/scripts/test.sh
@@ -0,0 +1,728 @@
+#!/bin/bash
+
+# eoAPI Test Suite
+# Combined Helm and Integration Testing Script
+# Supports both local development and CI environments
+
+# Source shared utilities
+SCRIPT_DIR="$(dirname "$(readlink -f "$0")")"
+source "$SCRIPT_DIR/lib/common.sh"
+
+# Global variables
+DEBUG_MODE=false
+NAMESPACE=""
+COMMAND=""
+
+# Auto-detect CI environment
+if is_ci_environment; then
+    DEBUG_MODE=true
+fi
+
+# Show help message
+show_help() {
+    cat << EOF
+eoAPI Test Suite - Combined Helm and Integration Testing
+
+USAGE:
+    $(basename "$0") [COMMAND] [OPTIONS]
+
+COMMANDS:
+    helm              Run Helm tests only (lint, unit tests, template validation)
+    integration       Run integration tests only (requires deployed eoAPI)
+    all               Run both Helm and integration tests [default]
+    check-deps        Check and install dependencies only
+    check-deployment  Check eoAPI deployment status only
+
+OPTIONS:
+    --debug           Enable debug mode with enhanced logging and diagnostics
+    --help, -h        Show this help message
+
+DESCRIPTION:
+    This script provides comprehensive testing for eoAPI:
+
+    Helm Tests:
+    - Chart linting with strict validation
+    - Helm unit tests (if test files exist)
+    - Template validation and rendering
+    - Kubernetes manifest validation (if kubeval available)
+
+    Integration Tests:
+    - Deployment verification
+    - Service readiness checks
+    - API endpoint testing
+    - Comprehensive failure debugging
+
+REQUIREMENTS:
+    Helm Tests: helm, helm unittest plugin
+    Integration Tests: kubectl, python/pytest, deployed eoAPI instance
+
+ENVIRONMENT VARIABLES:
+    RELEASE_NAME             Override release name detection
+    STAC_ENDPOINT            Override STAC API endpoint
+    RASTER_ENDPOINT          Override Raster API endpoint
+    VECTOR_ENDPOINT          Override Vector API endpoint
+
+    CI                       Auto-enables debug mode if set
+
+EXAMPLES:
+    $(basename "$0")                    # Run all tests
+    $(basename "$0") helm               # Run only Helm tests
+    $(basename "$0") integration        # Run only integration tests
+    $(basename "$0") check-deps         # Check dependencies only
+    $(basename "$0") check-deployment   # Check deployment status only
+    $(basename "$0") all --debug        # Run all tests with debug output
+    $(basename "$0") integration --debug # Run integration tests with enhanced logging
+    $(basename "$0") --help             # Show this help
+
+EOF
+}
+
+# Parse command line arguments
+parse_args() {
+    while [[ $# -gt 0 ]]; do
+        case $1 in
+            helm|integration|all|check-deps|check-deployment)
+                COMMAND="$1"; shift ;;
+            --debug)
+                DEBUG_MODE=true; shift ;;
+            --help|-h)
+                show_help; exit 0 ;;
+            *)
+                log_error "Unknown option: $1"
+                show_help; exit 1 ;;
+        esac
+    done
+
+    # Default to 'all' if no command specified
+    if [ -z "$COMMAND" ]; then
+        COMMAND="all"
+    fi
+}
+
+# Command exists function is now in common.sh
+
+# Check dependencies for helm tests
+check_helm_dependencies() {
+    preflight_test "helm" || exit 1
+
+    # Install unittest plugin if needed
+    if ! helm plugin list | grep -q unittest; then
+        log_info "Installing helm unittest plugin..."
+        helm plugin install https://github.com/helm-unittest/helm-unittest
+    fi
+}
+
+# Check dependencies for integration tests
+check_integration_dependencies() {
+    preflight_test "integration" || exit 1
+}
+
+# Install Python test dependencies
+install_test_deps() {
+    log_info "Installing Python test dependencies..."
+
+    local python_cmd="python"
+    if command_exists python3; then
+        python_cmd="python3"
+    fi
+
+    if ! $python_cmd -m pip install --quiet pytest httpx >/dev/null 2>&1; then
+        log_error "Failed to install test dependencies (pytest, httpx)"
+        log_error "Please install manually: pip install pytest httpx"
+        exit 1
+    fi
+
+    log_info "Test dependencies installed."
+}
+
+# Run Helm tests
+run_helm_tests() {
+    log_info "=== Running Helm Tests ==="
+
+    local failed_charts=()
+
+    # Run tests for each chart
+    for chart in charts/*/; do
+        if [ -f "$chart/Chart.yaml" ]; then
+            chart_name=$(basename "$chart")
+            log_info "Testing chart: $chart_name"
+
+            # 1. Helm lint with dependencies
+            log_info "  → Linting $chart_name..."
+            if ! helm lint "$chart" --strict; then
+                log_error "Linting failed for $chart_name"
+                failed_charts+=("$chart_name")
+                continue
+            fi
+
+            # 2. Helm unit tests (if test files exist)
+            if find "$chart" -name "*.yaml" -path "*/tests/*" | grep -q .; then
+                log_info "  → Running unit tests for $chart_name..."
+                if ! helm unittest "$chart" -f "tests/*.yaml"; then
+                    log_error "Unit tests failed for $chart_name"
+                    failed_charts+=("$chart_name")
+                    continue
+                fi
+            fi
+
+            # 3. Template validation
+            log_info "  → Validating templates for $chart_name..."
+            if ! helm template test-release "$chart" --dry-run > /dev/null; then
+                log_error "Template validation failed for $chart_name"
+                failed_charts+=("$chart_name")
+                continue
+            fi
+
+            # 4. K8s manifest validation (if kubeval available)
+            if command_exists kubeval; then
+                log_info "  → Validating K8s manifests for $chart_name..."
+                if ! helm template test-release "$chart" | kubeval; then
+                    log_error "K8s manifest validation failed for $chart_name"
+                    failed_charts+=("$chart_name")
+                    continue
+                fi
+            fi
+
+            log_info "  ✅ $chart_name tests passed"
+        fi
+    done
+
+    if [ ${#failed_charts[@]} -ne 0 ]; then
+        log_error "Helm tests failed for charts: ${failed_charts[*]}"
+        exit 1
+    fi
+
+    log_info "✅ All Helm tests passed"
+}
+
+# Check cluster connectivity
+check_cluster() {
+    validate_cluster || exit 1
+}
+
+# Detect release name and namespace from existing deployment
+detect_deployment() {
+    # Use environment variable if provided
+    if [ -n "${RELEASE_NAME:-}" ]; then
+        log_info "Using release name from environment: $RELEASE_NAME"
+    else
+        RELEASE_NAME=$(detect_release_name)
+        log_info "Detected release name: $RELEASE_NAME"
+        export RELEASE_NAME
+    fi
+
+    # Detect namespace
+    if [ -z "$NAMESPACE" ]; then
+        NAMESPACE=$(detect_namespace)
+        log_info "Detected namespace: $NAMESPACE"
+        export NAMESPACE
+    else
+        log_info "Using namespace from environment: $NAMESPACE"
+    fi
+}
+
+# Show debug information
+show_debug_info() {
+    log_info "=== Enhanced Debug Information ==="
+
+    log_info "=== Current Pod Status ==="
+    kubectl get pods -n "$NAMESPACE" -o wide || true
+
+    log_info "=== Pod Phase Summary ==="
+    kubectl get pods -n "$NAMESPACE" --no-headers | awk '{print $3}' | sort | uniq -c || true
+
+    log_info "=== Services Status ==="
+    kubectl get services -n "$NAMESPACE" || true
+
+    log_info "=== Ingress Status ==="
+    kubectl get ingress -n "$NAMESPACE" || true
+
+    log_info "=== Jobs Status ==="
+    kubectl get jobs -n "$NAMESPACE" -o wide || true
+
+    log_info "=== PostgreSQL Status ==="
+    kubectl get postgrescluster -o wide || true
+    kubectl get pods -l postgres-operator.crunchydata.com/cluster -o wide || true
+
+    log_info "=== Recent Events ==="
+    kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' | tail -30 || true
+}
+
+# Check if eoapi is deployed
+check_eoapi_deployment() {
+    if ! validate_eoapi_deployment "$NAMESPACE" "$RELEASE_NAME"; then
+        if [ "$DEBUG_MODE" = true ]; then
+            show_debug_info
+        else
+            log_info "You can deploy eoAPI using: make deploy or ./scripts/deploy.sh"
+        fi
+        exit 1
+    fi
+}
+
+# Wait for services to be ready
+wait_for_services() {
+    log_info "Waiting for services to be ready..."
+
+    # Function to wait for service with fallback label patterns
+    wait_for_service() {
+        local SERVICE=$1
+        log_info "Waiting for $SERVICE service to be ready..."
+
+        # Try multiple label patterns in order of preference
+        local PATTERNS=(
+            "app.kubernetes.io/instance=$RELEASE_NAME,app.kubernetes.io/name=$SERVICE"
+            "app=$RELEASE_NAME-$SERVICE"
+            "app.kubernetes.io/name=$SERVICE"
+        )
+
+        local FOUND_PODS=""
+        for PATTERN in "${PATTERNS[@]}"; do
+            FOUND_PODS=$(kubectl get pods -n "$NAMESPACE" -l "$PATTERN" -o name 2>/dev/null)
+            if [ -n "$FOUND_PODS" ]; then
+                log_debug "Found $SERVICE pods using pattern: $PATTERN"
+                kubectl get pods -n "$NAMESPACE" -l "$PATTERN" -o wide
+                if kubectl wait --for=condition=Ready pod -l "$PATTERN" -n "$NAMESPACE" --timeout=180s 2>/dev/null; then
+                    return 0
+                else
+                    log_warn "$SERVICE pods found but failed readiness check"
+                    kubectl describe pods -n "$NAMESPACE" -l "$PATTERN" 2>/dev/null || true
+                    return 1
+                fi
+            fi
+        done
+
+        # Fallback: find by pod name pattern
+        POD_NAME=$(kubectl get pods -n "$NAMESPACE" -o name | grep "$RELEASE_NAME-$SERVICE" | head -1)
+        if [ -n "$POD_NAME" ]; then
+            log_debug "Found $SERVICE pod by name pattern: $POD_NAME"
+            kubectl get "$POD_NAME" -n "$NAMESPACE" -o wide
+            if kubectl wait --for=condition=Ready "$POD_NAME" -n "$NAMESPACE" --timeout=180s 2>/dev/null; then
+                return 0
+            else
+                log_warn "$SERVICE pod found but failed readiness check"
+                kubectl describe "$POD_NAME" -n "$NAMESPACE" 2>/dev/null || true
+                return 1
+            fi
+        fi
+
+        log_error "No $SERVICE pods found with any pattern"
+        return 1
+    }
+
+    # Wait for each service
+    local failed_services=()
+    for service in raster vector stac; do
+        if ! wait_for_service "$service"; then
+            failed_services+=("$service")
+        fi
+    done
+
+    if [ ${#failed_services[@]} -ne 0 ]; then
+        log_error "Failed to start services: ${failed_services[*]}"
+
+        # Show debugging info
+        log_info "=== Debugging service startup failures ==="
+        kubectl get pods -n "$NAMESPACE" -o wide 2>/dev/null || true
+        kubectl get jobs -n "$NAMESPACE" -o wide 2>/dev/null || true
+        kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || true
+
+        exit 1
+    fi
+
+    log_info "All services are ready!"
+}
+
+# Setup port forwarding for localhost access
+setup_port_forwarding() {
+    local release_name="$1"
+
+    log_info "Setting up port forwarding for localhost access..."
+
+    # Kill any existing port forwards to avoid conflicts
+    pkill -f "kubectl port-forward.*$release_name" 2>/dev/null || true
+
+    # Wait a moment for processes to clean up
+    sleep 2
+
+    # Set up port forwarding in background
+    kubectl port-forward svc/"$release_name"-stac 8080:8080 -n "$NAMESPACE" >/dev/null 2>&1 &
+    local stac_pid=$!
+
+    kubectl port-forward svc/"$release_name"-raster 8081:8080 -n "$NAMESPACE" >/dev/null 2>&1 &
+    local raster_pid=$!
+
+    kubectl port-forward svc/"$release_name"-vector 8082:8080 -n "$NAMESPACE" >/dev/null 2>&1 &
+    local vector_pid=$!
+
+    # Give port forwards time to establish
+    sleep 3
+
+    # Check if port forwards are working
+    local failed_services=()
+
+    if ! netstat -ln 2>/dev/null | grep -q ":8080 "; then
+        failed_services+=("stac")
+        kill $stac_pid 2>/dev/null || true
+    fi
+
+    if ! netstat -ln 2>/dev/null | grep -q ":8081 "; then
+        failed_services+=("raster")
+        kill $raster_pid 2>/dev/null || true
+    fi
+
+    if ! netstat -ln 2>/dev/null | grep -q ":8082 "; then
+        failed_services+=("vector")
+        kill $vector_pid 2>/dev/null || true
+    fi
+
+    if [ ${#failed_services[@]} -eq 0 ]; then
+        log_info "Port forwarding established successfully"
+        # Update endpoints to use forwarded ports
+        export STAC_ENDPOINT="http://127.0.0.1:8080/stac"
+        export RASTER_ENDPOINT="http://127.0.0.1:8081/raster"
+        export VECTOR_ENDPOINT="http://127.0.0.1:8082/vector"
+
+        # Store PIDs for cleanup
+        echo "$stac_pid $raster_pid $vector_pid" > /tmp/eoapi-port-forward-pids
+
+        return 0
+    else
+        log_warn "Port forwarding failed for: ${failed_services[*]}"
+        return 1
+    fi
+}
+
+# Setup test environment
+setup_test_environment() {
+    # Use environment variables if already provided
+    if [ -n "${STAC_ENDPOINT:-}" ] && [ -n "${RASTER_ENDPOINT:-}" ] && [ -n "${VECTOR_ENDPOINT:-}" ]; then
+        log_info "Using endpoints from environment variables:"
+        log_info "  STAC: $STAC_ENDPOINT"
+        log_info "  Raster: $RASTER_ENDPOINT"
+        log_info "  Vector: $VECTOR_ENDPOINT"
+        return 0
+    fi
+
+    log_info "Setting up test environment..."
+
+    # Try to get the Traefik service IP (K3s pattern)
+    local publicip_value
+    publicip_value=$(kubectl -n kube-system get svc traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
+
+    # Fallback to other ingress controllers
+    if [ -z "$publicip_value" ]; then
+        publicip_value=$(kubectl get svc -n ingress-nginx ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
+    fi
+
+    # Try to get IP from ingress resources directly (works with minikube/nginx-ingress)
+    if [ -z "$publicip_value" ] && [ -n "$NAMESPACE" ]; then
+        publicip_value=$(kubectl get ingress -n "$NAMESPACE" -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
+    fi
+
+    # Fallback to check ingress in all namespaces
+    if [ -z "$publicip_value" ]; then
+        publicip_value=$(kubectl get ingress --all-namespaces -o jsonpath='{.items[0].status.loadBalancer.ingress[0].ip}' 2>/dev/null || echo "")
+    fi
+
+    # Try to get external IP from ingress controller service (works in many cloud CI environments)
+    if [ -z "$publicip_value" ]; then
+        publicip_value=$(kubectl get svc -n ingress-nginx ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].hostname}' 2>/dev/null || echo "")
+    fi
+
+    # Check for kind cluster (common in CI)
+    if [ -z "$publicip_value" ] && command_exists kind; then
+        if kind get clusters 2>/dev/null | grep -q .; then
+            # Kind typically uses localhost with port mapping
+            publicip_value="127.0.0.1"
+        fi
+    fi
+
+    # Try to get Docker Desktop IP (common in local development)
+    if [ -z "$publicip_value" ] && command_exists docker; then
+        if docker info 2>/dev/null | grep -q "Docker Desktop"; then
+            publicip_value="127.0.0.1"
+        fi
+    fi
+
+    # Try to get node external IP for bare metal clusters
+    if [ -z "$publicip_value" ]; then
+        publicip_value=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="ExternalIP")].address}' 2>/dev/null || echo "")
+    fi
+
+    # Fallback to node internal IP for bare metal/CI clusters
+    if [ -z "$publicip_value" ]; then
+        publicip_value=$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || echo "")
+    fi
+
+    # Try to get minikube IP if available
+    if [ -z "$publicip_value" ] && command_exists minikube; then
+        publicip_value=$(minikube ip 2>/dev/null || echo "")
+    fi
+
+    # Check for common CI environments and use localhost
+    if [ -z "$publicip_value" ] && [ -n "$CI" ]; then
+        # In many CI environments, services are accessible via localhost with port forwarding
+        publicip_value="127.0.0.1"
+    fi
+
+    # Try to get ingress host
+    local ingress_host=""
+    if [ -n "$NAMESPACE" ]; then
+        ingress_host=$(kubectl get ingress -n "$NAMESPACE" -o jsonpath='{.items[0].spec.rules[0].host}' 2>/dev/null || echo "")
+    fi
+    if [ -z "$ingress_host" ]; then
+        ingress_host=$(kubectl get ingress -o jsonpath='{.items[0].spec.rules[0].host}' 2>/dev/null || echo "")
+    fi
+
+    if [ "$DEBUG_MODE" = true ]; then
+        log_info "=== Debug Mode: Enhanced endpoint detection ==="
+        log_info "Ingress IP: $publicip_value"
+        log_info "Ingress Host: $ingress_host"
+    fi
+
+    # Set up endpoints
+    if [ -n "$publicip_value" ] && [ -n "$ingress_host" ]; then
+        log_info "Found ingress IP: $publicip_value, host: $ingress_host"
+
+        # Add to /etc/hosts if not already there
+        if ! grep -q "$ingress_host" /etc/hosts 2>/dev/null; then
+            if [ -w /etc/hosts ]; then
+                echo "$publicip_value $ingress_host" >> /etc/hosts
+                log_info "Added $ingress_host to /etc/hosts"
+            elif command_exists sudo; then
+                echo "$publicip_value $ingress_host" | sudo tee -a /etc/hosts >/dev/null
+                log_info "Added $ingress_host to /etc/hosts (with sudo)"
+            else
+                log_warn "Cannot write to /etc/hosts - you may need to add '$publicip_value $ingress_host' manually"
+            fi
+        fi
+
+        # Set endpoint environment variables
+        export VECTOR_ENDPOINT="http://$ingress_host/vector"
+        export STAC_ENDPOINT="http://$ingress_host/stac"
+        export RASTER_ENDPOINT="http://$ingress_host/raster"
+
+    elif [ -n "$publicip_value" ]; then
+        log_info "Found ingress IP: $publicip_value"
+        export VECTOR_ENDPOINT="http://$publicip_value/vector"
+        export STAC_ENDPOINT="http://$publicip_value/stac"
+        export RASTER_ENDPOINT="http://$publicip_value/raster"
+
+    else
+        log_warn "No external ingress found, attempting to use localhost with port forwarding"
+
+        # Try to set up automatic port forwarding
+        if setup_port_forwarding "$RELEASE_NAME"; then
+            log_info "Successfully configured localhost access via port forwarding"
+        else
+            log_warn "Automatic port forwarding failed, using direct endpoints"
+            log_warn "You may need to manually set up port forwarding:"
+            log_warn "kubectl port-forward svc/$RELEASE_NAME-stac 8080:8080 -n $NAMESPACE &"
+            log_warn "kubectl port-forward svc/$RELEASE_NAME-raster 8081:8080 -n $NAMESPACE &"
+            log_warn "kubectl port-forward svc/$RELEASE_NAME-vector 8082:8080 -n $NAMESPACE &"
+
+            # Fallback to direct endpoints (may not work)
+            export VECTOR_ENDPOINT="http://127.0.0.1/vector"
+            export STAC_ENDPOINT="http://127.0.0.1/stac"
+            export RASTER_ENDPOINT="http://127.0.0.1/raster"
+        fi
+    fi
+
+    log_info "Service endpoints configured:"
+    log_info "  STAC: $STAC_ENDPOINT"
+    log_info "  Raster: $RASTER_ENDPOINT"
+    log_info "  Vector: $VECTOR_ENDPOINT"
+}
+
+# Run integration tests
+run_integration_tests() {
+    log_info "=== Running Integration Tests ==="
+
+    local python_cmd="python"
+    if command_exists python3; then
+        python_cmd="python3"
+    fi
+
+    local test_dir=".github/workflows/tests"
+    if [ ! -d "$test_dir" ]; then
+        log_error "Test directory not found: $test_dir"
+        exit 1
+    fi
+
+    log_info "Test environment:"
+    log_info "  STAC_ENDPOINT=${STAC_ENDPOINT:-[not set]}"
+    log_info "  RASTER_ENDPOINT=${RASTER_ENDPOINT:-[not set]}"
+    log_info "  VECTOR_ENDPOINT=${VECTOR_ENDPOINT:-[not set]}"
+
+    # Run tests individually with error handling
+    local failed_tests=()
+
+    # Vector tests
+    log_info "=== Running vector tests ==="
+    if ! $python_cmd -m pytest "$test_dir/test_vector.py" -v; then
+        log_error "Vector tests failed"
+        failed_tests+=("vector")
+
+        # Show service logs on failure
+        log_info "=== Vector service logs ==="
+        kubectl logs svc/"$RELEASE_NAME"-vector -n "$NAMESPACE" --tail=50 2>/dev/null || \
+        kubectl logs deployment/"$RELEASE_NAME"-vector -n "$NAMESPACE" --tail=50 2>/dev/null || \
+        log_warn "Could not get vector service logs"
+    else
+        log_info "Vector tests passed"
+    fi
+
+    # STAC tests
+    log_info "=== Running STAC tests ==="
+    if ! $python_cmd -m pytest "$test_dir/test_stac.py" -v; then
+        log_error "STAC tests failed"
+        failed_tests+=("stac")
+
+        # Show service logs on failure
+        log_info "=== STAC service logs ==="
+        kubectl logs svc/"$RELEASE_NAME"-stac -n "$NAMESPACE" --tail=50 2>/dev/null || \
+        kubectl logs deployment/"$RELEASE_NAME"-stac -n "$NAMESPACE" --tail=50 2>/dev/null || \
+        log_warn "Could not get STAC service logs"
+    else
+        log_info "STAC tests passed"
+    fi
+
+    # Raster tests
+    log_info "=== Running raster tests ==="
+    if ! $python_cmd -m pytest "$test_dir/test_raster.py" -v; then
+        log_warn "Raster tests failed (known to be flaky)"
+        failed_tests+=("raster")
+
+        # Show service logs on failure
+        log_info "=== Raster service logs ==="
+        kubectl logs svc/"$RELEASE_NAME"-raster -n "$NAMESPACE" --tail=50 2>/dev/null || \
+        kubectl logs deployment/"$RELEASE_NAME"-raster -n "$NAMESPACE" --tail=50 2>/dev/null || \
+        log_warn "Could not get raster service logs"
+    else
+        log_info "Raster tests passed"
+    fi
+
+    # Report results
+    if [ ${#failed_tests[@]} -eq 0 ]; then
+        log_info "✅ All integration tests completed successfully!"
+    else
+        log_error "Some tests failed: ${failed_tests[*]}"
+
+        # Comprehensive debugging
+        log_info "=== Final Deployment Status ==="
+        kubectl get pods -n "$NAMESPACE" -o wide 2>/dev/null || true
+        kubectl get services -n "$NAMESPACE" 2>/dev/null || true
+        kubectl get events -n "$NAMESPACE" --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || true
+
+        # Only fail if critical tests (vector/stac) failed
+        if [[ " ${failed_tests[*]} " =~ " vector " ]] || [[ " ${failed_tests[*]} " =~ " stac " ]]; then
+            exit 1
+        else
+            log_warn "Only raster tests failed (known issue), continuing..."
+        fi
+    fi
+}
+
+# Main function
+main() {
+    parse_args "$@"
+
+    if [ "$DEBUG_MODE" = true ]; then
+        log_info "Starting eoAPI test suite (DEBUG MODE) - Command: $COMMAND"
+    else
+        log_info "Starting eoAPI test suite - Command: $COMMAND"
+    fi
+
+    # Run tests based on command
+    case $COMMAND in
+        helm)
+            check_helm_dependencies
+            run_helm_tests
+            ;;
+        check-deps)
+            log_info "Checking all dependencies..."
+            check_helm_dependencies
+            check_integration_dependencies
+            check_cluster
+            install_test_deps
+            log_info "✅ All dependencies checked and ready"
+            ;;
+        check-deployment)
+            log_info "Checking deployment status..."
+            check_integration_dependencies
+            check_cluster
+            detect_deployment
+            check_eoapi_deployment
+            log_info "✅ Deployment check complete"
+            ;;
+        integration)
+            check_integration_dependencies
+            check_cluster
+            install_test_deps
+            detect_deployment
+
+            # Show enhanced debugging in debug mode
+            if [ "$DEBUG_MODE" = true ]; then
+                show_debug_info
+            fi
+
+            check_eoapi_deployment
+
+            wait_for_services
+            setup_test_environment
+
+            run_integration_tests
+            ;;
+        all)
+            log_info "Running comprehensive test suite (Helm + Integration tests)"
+
+            # Run Helm tests first
+            log_info "=== Phase 1: Helm Tests ==="
+            check_helm_dependencies
+            run_helm_tests
+
+            # Run Integration tests second
+            log_info "=== Phase 2: Integration Tests ==="
+            check_integration_dependencies
+            check_cluster
+            install_test_deps
+            detect_deployment
+
+            # Show enhanced debugging in debug mode
+            if [ "$DEBUG_MODE" = true ]; then
+                show_debug_info
+            fi
+
+            check_eoapi_deployment
+
+            wait_for_services
+            setup_test_environment
+
+            run_integration_tests
+            ;;
+        *)
+            log_error "Unknown command: $COMMAND"
+            show_help
+            exit 1
+            ;;
+    esac
+
+    # Clean up port forwarding if it was set up
+    if [ -f /tmp/eoapi-port-forward-pids ]; then
+        log_info "Cleaning up port forwarding..."
+        while read -r pid; do
+            kill "$pid" 2>/dev/null || true
+        done < /tmp/eoapi-port-forward-pids
+        rm -f /tmp/eoapi-port-forward-pids
+    fi
+
+    if [ "$DEBUG_MODE" = true ]; then
+        log_info "eoAPI test suite complete (DEBUG MODE)!"
+    else
+        log_info "eoAPI test suite complete!"
+    fi
+}
+
+# Run main function
+main "$@"

From 275606fbe5dde13ef5097fc745d087105dfdfd4a Mon Sep 17 00:00:00 2001
From: Felix Delattre <felix@developmentseed.org>
Date: Wed, 10 Sep 2025 09:22:01 +0200
Subject: [PATCH 2/4] Checked on raster and vector integration tests.

---
 .github/workflows/tests/test_raster.py | 14 ++++++-------
 .github/workflows/tests/test_vector.py | 28 ++++++++++++++++++--------
 2 files changed, 27 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/tests/test_raster.py b/.github/workflows/tests/test_raster.py
index b483a126..8daae4f5 100644
--- a/.github/workflows/tests/test_raster.py
+++ b/.github/workflows/tests/test_raster.py
@@ -30,14 +30,14 @@ def test_mosaic_api(raster_endpoint):
 
     searchid = resp.json()["id"]
 
-    resp = client.get(f"{raster_endpoint}/searches/{searchid}/-85.6358,36.1624/assets")
+    resp = client.get(f"{raster_endpoint}/searches/{searchid}/point/-85.6358,36.1624/assets")
     assert resp.status_code == 200
     assert len(resp.json()) == 1
     assert list(resp.json()[0]) == ["id", "bbox", "assets", "collection"]
     assert resp.json()[0]["id"] == "20200307aC0853900w361030"
 
     resp = client.get(
-        f"{raster_endpoint}/searches/{searchid}/tiles/15/8589/12849/assets"
+        f"{raster_endpoint}/searches/{searchid}/tiles/WebMercatorQuad/15/8589/12849/assets"
     )
     assert resp.status_code == 200
     assert len(resp.json()) == 1
@@ -46,7 +46,7 @@ def test_mosaic_api(raster_endpoint):
 
     z, x, y = 15, 8589, 12849
     resp = client.get(
-        f"{raster_endpoint}/searches/{searchid}/tiles/{z}/{x}/{y}",
+        f"{raster_endpoint}/searches/{searchid}/tiles/WebMercatorQuad/{z}/{x}/{y}",
         params={"assets": "cog"},
         headers={"Accept-Encoding": "br, gzip"},
         timeout=10.0,
@@ -59,7 +59,7 @@ def test_mosaic_api(raster_endpoint):
 def test_mosaic_collection_api(raster_endpoint):
     """test mosaic collection."""
     resp = client.get(
-        f"{raster_endpoint}/collections/noaa-emergency-response/-85.6358,36.1624/assets"
+        f"{raster_endpoint}/collections/noaa-emergency-response/point/-85.6358,36.1624/assets"
     )
     assert resp.status_code == 200
     assert len(resp.json()) == 1
@@ -67,7 +67,7 @@ def test_mosaic_collection_api(raster_endpoint):
     assert resp.json()[0]["id"] == "20200307aC0853900w361030"
 
     resp = client.get(
-        f"{raster_endpoint}/collections/noaa-emergency-response/tiles/15/8589/12849/assets"
+        f"{raster_endpoint}/collections/noaa-emergency-response/tiles/WebMercatorQuad/15/8589/12849/assets"
     )
     assert resp.status_code == 200
     assert len(resp.json()) == 1
@@ -76,7 +76,7 @@ def test_mosaic_collection_api(raster_endpoint):
 
     z, x, y = 15, 8589, 12849
     resp = client.get(
-        f"{raster_endpoint}/collections/noaa-emergency-response/tiles/{z}/{x}/{y}",
+        f"{raster_endpoint}/collections/noaa-emergency-response/tiles/WebMercatorQuad/{z}/{x}/{y}",
         params={"assets": "cog"},
         headers={"Accept-Encoding": "br, gzip"},
         timeout=10.0,
@@ -214,7 +214,7 @@ def test_item(raster_endpoint):
     assert resp.json() == ["cog"]
 
     resp = client.get(
-        f"{raster_endpoint}/collections/noaa-emergency-response/items/20200307aC0853300w361200/tilejson.json",
+        f"{raster_endpoint}/collections/noaa-emergency-response/items/20200307aC0853300w361200/WebMercatorQuad/tilejson.json",
         params={
             "assets": "cog",
         },
diff --git a/.github/workflows/tests/test_vector.py b/.github/workflows/tests/test_vector.py
index d5f0d7b5..0216db34 100644
--- a/.github/workflows/tests/test_vector.py
+++ b/.github/workflows/tests/test_vector.py
@@ -39,21 +39,33 @@ def test_vector_api(vector_endpoint):
         "collections",
     ]
 
-    total_timeout = 60 * 5
+    total_timeout = 60 * 2
     start_time = time.time()
     while True:
-        if resp.json()["numberMatched"] == 7:
+        collections_data = resp.json()
+        current_count = collections_data.get("numberMatched", 0)
+        print(f"Current collections count: {current_count}/7")
+
+        if current_count == 7:
             break
 
-        if time.time() - start_time > total_timeout:
-            print("Timeout exceeded")
-            assert False
+        elapsed_time = time.time() - start_time
+        if elapsed_time > total_timeout:
+            print(f"Timeout exceeded after {elapsed_time:.1f}s. Expected 7 collections, got {current_count}")
+            if "collections" in collections_data:
+                available_collections = [c.get("id", "unknown") for c in collections_data["collections"]]
+                print(f"Available collections: {available_collections}")
+            assert False, f"Expected 7 collections but found {current_count} after {elapsed_time:.1f}s timeout"
 
-        time.sleep(20)
+        time.sleep(10)
         resp = client.get(f"{vector_endpoint}/collections")
 
-    assert resp.json()["numberMatched"] == 7  # one public table + 5 functions
-    assert resp.json()["numberReturned"] == 7
+    collections_data = resp.json()
+    matched_count = collections_data.get("numberMatched", 0)
+    returned_count = collections_data.get("numberReturned", 0)
+
+    assert matched_count == 7, f"Expected 7 matched collections, got {matched_count}. Available: {[c.get('id', 'unknown') for c in collections_data.get('collections', [])]}"
+    assert returned_count == 7, f"Expected 7 returned collections, got {returned_count}"
 
     collections = resp.json()["collections"]
     ids = [c["id"] for c in collections]

From 2d9a0db13fd9bdee6ef1f524b83bf9c3d9a6fe70 Mon Sep 17 00:00:00 2001
From: Felix Delattre <felix@developmentseed.org>
Date: Wed, 10 Sep 2025 09:22:48 +0200
Subject: [PATCH 3/4] Run CI integration tests with deploy and test scripts.

---
 .github/workflows/helm-tests.yml | 412 ++++++-------------------------
 .gitignore                       |   1 +
 2 files changed, 83 insertions(+), 330 deletions(-)

diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index 9fd5b56b..5e1fbdfe 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -12,379 +12,131 @@ env:
   PGO_VERSION: 5.7.4
 
 jobs:
-  helm-tests:
+  test:
+    name: Helm tests
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
-      - uses: d3adb5/helm-unittest-action@v2
+      - name: Install Helm
+        uses: azure/setup-helm@v3
         with:
-          helm-version: ${{ env.HELM_VERSION }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
+          version: ${{ env.HELM_VERSION }}
+
+      - name: Run Helm unit tests
+        run: make tests
 
-      - run: |
-          cd charts
-          helm unittest eoapi -f 'tests/*.yaml' -v eoapi/test-helm-values.yaml
-  k3s-integration-tests:
-    if: github.event.pull_request.head.repo.full_name == github.repository
+  integration:
+    name: Integration Tests (K3s)
+    # if: github.event.pull_request.head.repo.full_name == github.repository
     permissions:
       contents: 'read'
       id-token: 'write'
-    needs: helm-tests
+    needs: test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
 
-      - name: Start a local k3s cluster
+      - name: Start K3s cluster
         uses: jupyterhub/action-k3s-helm@v4
         with:
-          # See available:
-          # - k3s release channels at https://github.com/k3s-io/k3s/blob/HEAD/channel.yaml
-          # - k3s versions at https://github.com/k3s-io/k3s/tags
-          # - helm versions at https://github.com/helm/helm/tags
           k3s-channel: latest
           helm-version: ${{ env.HELM_VERSION }}
           metrics-enabled: false
           docker-enabled: true
 
-      - name: last commit sha if PR
-        if: ${{ github.event_name == 'pull_request' }}
-        shell: bash
-        run: |
-          echo "LAST_COMMIT_SHA=${{ github.event.pull_request.head.sha }}" >> ${GITHUB_ENV}
-
-      - name: last commit sha if push
-        if: ${{ github.event_name == 'push' }}
-        shell: bash
+      - name: Set release name
         run: |
-          echo "LAST_COMMIT_SHA=${GITHUB_SHA}" >> ${GITHUB_ENV}
+          SHORT_SHA="${{ github.sha }}"
+          SHORT_SHA="${SHORT_SHA::8}"
+          SALT=$(echo "$RANDOM" | cut -c1-3)
+          echo "RELEASE_NAME=eoapi-$SHORT_SHA-$SALT" >> "$GITHUB_ENV"
 
-      - name: set k8s .release.name suffix
-        run: |
-          # salt for randomness per test run
-          COMMITSHA=$(echo $LAST_COMMIT_SHA | cut -c 1-6)
-          SALT=$(echo "${RANDOM}${RANDOM}${RANDOM}" | cut -c1-3)
-          echo "RELEASE_NAME=eoapi$COMMITSHA$SALT" >> $GITHUB_ENV
-
-      - name: helm install crunchydata postgres operator
-        run: |
-          helm upgrade --install \
-            --set disable_check_for_upgrades=true \
-            pgo \
-            oci://registry.developers.crunchydata.com/crunchydata/pgo \
-            --version ${{ env.PGO_VERSION }}
-
-      - id: helm-render-install-eoapi-templates
-        name: helm render/install eoapi templates
+      - name: Deploy eoAPI
+        id: deploy
         continue-on-error: true
         run: |
-          export GITSHA='${{github.sha}}'
-
-          cd charts
-
-          helm dependency build eoapi
-
-          helm install $RELEASE_NAME \
-            -f ./eoapi/values.yaml \
-            -f ./eoapi/test-k3s-unittest-values.yaml \
-            ./eoapi
-
-          exit $?
-
-      - name: show jobs and pods
+          echo "=== Starting eoAPI deployment ==="
+          export RELEASE_NAME="$RELEASE_NAME"
+          export PGO_VERSION="${{ env.PGO_VERSION }}"
+          export GITHUB_SHA="${{ github.sha }}"
+          ./scripts/deploy.sh --ci
+
+      - name: Check deployment status
+        id: check
+        if: steps.deploy.outcome == 'success'
         run: |
-          echo "===== Jobs ====="
-          kubectl get jobs -o wide
-          echo ""
-          echo "===== Job Status Details ====="
-          kubectl get jobs -o custom-columns='NAME:.metadata.name,COMPLETIONS:.spec.completions,SUCCESSFUL:.status.succeeded,FAILED:.status.failed,AGE:.metadata.creationTimestamp'
-          echo ""
-          echo "===== All Pods ====="
-          kubectl get pods -o wide
-          echo ""
-          echo "===== Pods (pgstac) ====="
-          kubectl get pods | grep -i pgstac || true
-          echo ""
-          echo "===== Pod Phase Summary ====="
-          kubectl get pods --no-headers | awk '{print $3}' | sort | uniq -c
-          echo ""
-          echo "===== Events (last 10 minutes) ====="
-          kubectl get events --sort-by='.lastTimestamp' | tail -20
+          echo "=== Checking deployment status ==="
+          export RELEASE_NAME="$RELEASE_NAME"
+          ./scripts/test.sh check-deployment --debug
 
-      - name: debug pgstac-eoapi-superuser-init-db job failure
-        if: steps.helm-render-install-eoapi-templates.outcome == 'failure'
+      - name: Debug pgstac jobs if deployment failed
+        if: steps.deploy.outcome == 'failure'
         continue-on-error: true
         run: |
-          echo "Extracting pgstac-eoapi-superuser-init-db job info and logs for debugging..."
-
-          # Get job details
-          echo "===== pgstac-eoapi-superuser-init-db Job Details ====="
-          kubectl get job "$RELEASE_NAME-pgstac-eoapi-superuser-init-db" -o yaml || echo "Could not get job details"
-
-          # Get pod details
-          echo "===== Pod Details ====="
-          kubectl get pods --selector=app=pgstac-eoapi-superuser-init-db -o wide || echo "Could not find pods"
-
-          # Extract logs from pods
-          echo "===== Pod Logs ====="
-          PODS=$(kubectl get pods --selector=app=pgstac-eoapi-superuser-init-db -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
-          if [ -n "$PODS" ]; then
-            for POD in $PODS; do
-              echo "--- Logs from pod $POD ---"
-              kubectl logs $POD --previous || true  # Get logs from previous container if it exists
-              kubectl logs $POD || echo "Could not get logs from pod $POD"
+          echo "=== Debugging pgstac job failures ==="
+
+          # Check pgstac-migrate job
+          echo "===== pgstac-migrate Job Status ====="
+          kubectl get jobs -l app.kubernetes.io/name=pgstac-migrate -o wide || echo "No pgstac-migrate jobs found"
+
+          MIGRATE_PODS=$(kubectl get pods -l app.kubernetes.io/name=pgstac-migrate -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
+          if [ -n "$MIGRATE_PODS" ]; then
+            for POD in $MIGRATE_PODS; do
+              echo "--- Logs from migrate pod $POD ---"
+              kubectl logs "$POD" --tail=100 || true
+              echo "--- Description of migrate pod $POD ---"
+              kubectl describe pod "$POD"
             done
-          else
-            echo "No pods found for pgstac-eoapi-superuser-init-db job"
           fi
 
-          # Get pod descriptions for more details
-          echo "===== Pod Descriptions ====="
-          kubectl describe pods --selector=app=pgstac-eoapi-superuser-init-db || echo "Could not describe pods"
-
-          # Check the configmap contents
-          echo "===== initdb ConfigMap Contents ====="
-          kubectl get configmap initdb -o yaml || echo "Could not get initdb configmap"
-
-          # Check for any related events
-          echo "===== Related Kubernetes Events ====="
-          kubectl get events | grep -E "pgstac-eoapi-superuser-init-db|initdb" || echo "No relevant events found"
-
-      - name: debug pgstac-migrate job failure
-        if: steps.helm-render-install-eoapi-templates.outcome == 'failure'
-        continue-on-error: true
-        run: |
-          echo "Extracting comprehensive pgstac-migrate job info and logs for debugging..."
-
-          # Get all jobs with details
-          echo "===== All Jobs Status ====="
-          kubectl get jobs -o wide
-          echo ""
-
-          # Get specific job details using labels
-          echo "===== pgstac-migrate Job Details (by label) ====="
-          kubectl get jobs -l app=pgstac-migrate -o yaml || echo "Could not get pgstac-migrate job details"
-          echo ""
-
-          # Get pod details - both by label and by job-name
-          echo "===== pgstac-migrate Pod Details (by label) ====="
-          kubectl get pods -l app=pgstac-migrate --all-namespaces -o wide || echo "Could not find pgstac-migrate pods by label"
-          echo ""
-
-          echo "===== pgstac-migrate Pod Details (by app label) ====="
-          kubectl get pods -l app=pgstac-migrate -o wide || echo "Could not find pgstac-migrate pods by app label"
-          echo ""
-
-          # Extract logs from all pgstac-migrate pods (running, completed, failed)
-          echo "===== pgstac-migrate Pod Logs ====="
-          ALL_PODS=$(kubectl get pods -l app=pgstac-migrate -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
-          if [ -n "$ALL_PODS" ]; then
-            echo "Found pgstac-migrate job pods. Extracting logs from each:"
-            for POD in $ALL_PODS; do
-              echo "--- Pod $POD status ---"
-              kubectl get pod "$POD" -o wide
-              echo "--- Logs from pod $POD ---"
-              kubectl logs pod/$POD --tail=100 || echo "Could not get logs from pod $POD"
-              echo "--- Previous logs from pod $POD (if container restarted) ---"
-              kubectl logs pod/$POD --previous --tail=50 || echo "No previous logs for pod $POD"
-              echo ""
+          # Check pgstac-load-samples job
+          echo "===== pgstac-load-samples Job Status ====="
+          kubectl get jobs -l app.kubernetes.io/name=pgstac-load-samples -o wide || echo "No pgstac-load-samples jobs found"
+
+          SAMPLES_PODS=$(kubectl get pods -l app.kubernetes.io/name=pgstac-load-samples -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
+          if [ -n "$SAMPLES_PODS" ]; then
+            for POD in $SAMPLES_PODS; do
+              echo "--- Logs from samples pod $POD ---"
+              kubectl logs "$POD" --tail=100 || true
+              echo "--- Description of samples pod $POD ---"
+              kubectl describe pod "$POD"
             done
-          else
-            echo "No pods found for pgstac-migrate jobs"
-            echo "Checking for pods with broader label search..."
-            LABEL_PODS=$(kubectl get pods -l app=pgstac-migrate -o jsonpath='{.items[*].metadata.name}' 2>/dev/null)
-            if [ -n "$LABEL_PODS" ]; then
-              for POD in $LABEL_PODS; do
-                echo "--- Pod $POD (found by label) ---"
-                kubectl describe pod "$POD"
-                kubectl logs pod/$POD --tail=50 || true
-              done
-            fi
           fi
 
-          # Get details about the database pods/services
-          echo "===== Database Pod/Service Details ====="
-          kubectl get svc | grep -E "db|postgres" || echo "Could not find database services"
-          kubectl get pods | grep -E "db-|postgres" || echo "Could not find database pods"
-          echo ""
+          # Check database status
+          echo "===== Database Pod Status ====="
+          kubectl get pods -l postgres-operator.crunchydata.com/cluster -o wide
+          kubectl get postgrescluster -o wide
 
-          # Check ConfigMaps and Secrets
+          # Check ConfigMaps
           echo "===== Relevant ConfigMaps ====="
-          kubectl get configmaps | grep -E "pgstac|initdb" || echo "No pgstac configmaps found"
-          echo ""
+          kubectl get configmaps | grep -E "initdb|pgstac" || echo "No relevant configmaps found"
 
-          # Check for any events related to the job or pods
-          echo "===== Related Kubernetes Events (last 50) ====="
-          kubectl get events --sort-by='.lastTimestamp' | grep -E "pgstac|db|migrate" || echo "No relevant events found"
+          exit 1
 
-      - id: watchservices
-        name: watch services boot
-        timeout-minutes: 3
-        continue-on-error: true
+      - name: Run integration tests
+        if: steps.deploy.outcome == 'success'
         run: |
-          # Wait for services to be ready using native readiness checks
-          echo "===== Current Pod Status ====="
-          kubectl get pods -o wide
-          echo ""
-
-          echo "Waiting for raster service to be ready..."
-          kubectl wait --for=condition=Ready pod -l app=${RELEASE_NAME}-raster --timeout=180s || {
-            echo "Raster service failed to become ready. Checking status..."
-            kubectl get pods -l app=${RELEASE_NAME}-raster -o wide
-            kubectl describe pods -l app=${RELEASE_NAME}-raster
-            exit 1
-          }
-          echo "raster service is ready, moving on..."
+          echo "=== Running integration tests ==="
+          export RELEASE_NAME="$RELEASE_NAME"
+          ./scripts/test.sh integration --debug
 
-          echo "Waiting for vector service to be ready..."
-          kubectl wait --for=condition=Ready pod -l app=${RELEASE_NAME}-vector --timeout=180s || {
-            echo "Vector service failed to become ready. Checking status..."
-            kubectl get pods -l app=${RELEASE_NAME}-vector -o wide
-            kubectl describe pods -l app=${RELEASE_NAME}-vector
-            exit 1
-          }
-          echo "vector service is ready, moving on..."
-
-          echo "Waiting for stac service to be ready..."
-          kubectl wait --for=condition=Ready pod -l app=${RELEASE_NAME}-stac --timeout=180s || {
-            echo "STAC service failed to become ready. Checking status..."
-            kubectl get pods -l app=${RELEASE_NAME}-stac -o wide
-            kubectl describe pods -l app=${RELEASE_NAME}-stac
-            exit 1
-          }
-          echo "all services are ready, moving on..."
-
-      - name: cleanup if services fail to boot
-        if: steps.watchservices.outcome == 'failure'
+      - name: Debug deployment status
+        if: always()
         run: |
-          echo "The watchservices step failed or timed out. Extracting comprehensive debugging info..."
-
-          # Get and display all pods status with more detail
-          echo "===== Pod Status (detailed) ====="
+          echo "=== Final Deployment Status ==="
           kubectl get pods -o wide
-          echo ""
-          
-          echo "===== Pod Readiness Summary ====="
-          kubectl get pods --no-headers | awk '{print $2, $3}' | sort | uniq -c
-          echo ""
-
-          # Check init container logs for all services
-          for SERVICE in raster vector stac multidim; do
-            echo "===== $SERVICE Service Pod Status ====="
-            kubectl get pods -l app=$RELEASE_NAME-$SERVICE -o wide || echo "No $SERVICE pods found"
-            
-            POD_NAME=$(kubectl get pod -l app=$RELEASE_NAME-$SERVICE -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
-            if [ -n "$POD_NAME" ]; then
-              echo "===== $SERVICE Pod ($POD_NAME) Init Container Logs ====="
-              kubectl logs pod/$POD_NAME -c wait-for-pgstac-jobs --tail=100 || echo "Could not get $SERVICE init container logs"
-              echo ""
-              
-              echo "===== $SERVICE Pod ($POD_NAME) Main Container Logs ====="
-              kubectl logs pod/$POD_NAME --tail=100 || echo "Could not get $SERVICE main container logs"
-              echo ""
-              
-              echo "===== $SERVICE Pod ($POD_NAME) Description ====="
-              kubectl describe pod/$POD_NAME
-              echo ""
-            fi
-          done
-
-          # Show job status that init containers might be waiting for
-          echo "===== Job Status (what init containers are waiting for) ====="
-          kubectl get jobs -o wide
-          echo ""
-          
-          # Check pgstac jobs using labels instead of hardcoded names
-          for APP_LABEL in pgstac-migrate pgstac-load-samples; do
-            echo "===== Jobs with app=$RELEASE_NAME-$APP_LABEL Status ====="
-            JOBS=$(kubectl get jobs -l app=$RELEASE_NAME-$APP_LABEL -o name 2>/dev/null || true)
-            if [ -n "$JOBS" ]; then
-              for JOB in $JOBS; do
-                echo "--- Job $JOB ---"
-                kubectl get "$JOB" -o yaml 2>/dev/null | grep -A 10 -E "conditions|status:" || echo "Could not get status for $JOB"
-              done
-            else
-              echo "No jobs found with app=$RELEASE_NAME-$APP_LABEL label"
-            fi
-            echo ""
-          done
-
-          # Check recent events
-          echo "===== Recent Events (last 50) ====="
-          kubectl get events --sort-by='.lastTimestamp' | tail -50
-          echo ""
-
-          # force GH action to show failed result
-          exit 128
-
-      - name: install python unit-test dependencies
-        run: |
-          python -m pip install pytest httpx
+          kubectl get services
+          kubectl get ingress
 
-      - name: run the tests
-        id: testrunner
-        # continue-on-error: true
+      - name: Debug on failure
+        if: failure()
         run: |
-          kubectl get svc --all-namespaces
-          kubectl get ingress --all-namespaces -o jsonpath='{range .items[0]}kubectl describe ingress {.metadata.name} -n {.metadata.namespace}{end}' | sh
-          kubectl get middleware.traefik.io --all-namespaces -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name' --no-headers | while read -r namespace name; do kubectl describe middleware.traefik.io "$name" -n "$namespace"; done
-
-          # Get the IP address of the Traefik service
-          PUBLICIP_VALUE=$(kubectl -n kube-system get svc traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
-          PUBLICIP=http://eoapi.local
-          export VECTOR_ENDPOINT=$PUBLICIP/vector
-          export STAC_ENDPOINT=$PUBLICIP/stac
-          export RASTER_ENDPOINT=$PUBLICIP/raster
-
-          # Add entry to /etc/hosts for eoapi.local
-          echo "Adding eoapi.local to /etc/hosts with IP: $PUBLICIP_VALUE"
-          echo "$PUBLICIP_VALUE eoapi.local" | sudo tee -a /etc/hosts
-
-          echo '#################################'
-          echo $VECTOR_ENDPOINT
-          echo $STAC_ENDPOINT
-          echo $RASTER_ENDPOINT
-          echo '#################################'
-
-          # Run tests with proper failure propagation
-          set -e  # Make sure any command failure causes the script to exit with error
-          pytest .github/workflows/tests/test_vector.py || { kubectl logs svc/vector; exit 1; }
-          pytest .github/workflows/tests/test_stac.py || { kubectl logs svc/stac; exit 1; }
-          # TODO: fix raster tests
-          #pytest .github/workflows/tests/test_raster.py || { kubectl logs svc/raster; exit 1; }
-
-      - name: error if tests failed
-        if: steps.testrunner.outcome == 'failure'
-        run: |
-          echo "The tests failed. Extracting pod logs for debugging..."
-
-          # Get and display all pods status
-          echo "===== Pod Status ====="
-          kubectl get pods
-
-          # Extract logs from raster pod init container (wait-for-pgstac-jobs)
-          echo "===== Raster Pod Init Container Logs (wait-for-pgstac-jobs) ====="
-          kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} -c wait-for-pgstac-jobs --tail=100 || echo "Could not get raster init container logs"
-
-          # Extract logs from raster pod main container
-          echo "===== Raster Pod Main Container Logs ====="
-          kubectl get pod | grep "^raster-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get raster main container logs"
-
-          # Extract logs from vector pod
-          echo "===== Vector Pod Logs ====="
-          kubectl get pod | grep "^vector-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get vector logs"
-
-          # Extract logs from stac pod
-          echo "===== STAC Pod Logs ====="
-          kubectl get pod | grep "^stac-$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl logs pod/{} --tail=100 || echo "Could not get STAC logs"
-
-          # Check if pods are in pending state or have issues
-          echo "===== Pod Descriptions for Troubleshooting ====="
-          kubectl get pod | grep "$RELEASE_NAME" | cut -d' ' -f1 | xargs -I{} kubectl describe pod/{} || echo "Could not describe pods"
-
-          # force GH action to show failed result
-          exit 128
+          echo "=== Additional failure debugging (integration script provides comprehensive debugging) ==="
+          kubectl get events --sort-by='.lastTimestamp' | tail -20 || true
 
-      - name: helm uninstall eoapi templates
+      - name: Cleanup
+        if: always()
         run: |
-          helm uninstall $RELEASE_NAME
+          helm uninstall "$RELEASE_NAME" || true
diff --git a/.gitignore b/.gitignore
index 22414605..35f7b4e8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,3 +4,4 @@
 charts/config.yaml
 charts/eoapi/charts/*.tgz
 config_ingress.yaml
+__pycache__

From 2b51600bc380d578cb928485d3dd608f5922083e Mon Sep 17 00:00:00 2001
From: Felix Delattre <felix@developmentseed.org>
Date: Thu, 11 Sep 2025 09:46:39 +0200
Subject: [PATCH 4/4] Adjusted files based on PR review feedback.

---
 .github/workflows/helm-tests.yml | 5 ++---
 Makefile                         | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/helm-tests.yml b/.github/workflows/helm-tests.yml
index 5e1fbdfe..d646d262 100644
--- a/.github/workflows/helm-tests.yml
+++ b/.github/workflows/helm-tests.yml
@@ -28,7 +28,7 @@ jobs:
 
   integration:
     name: Integration Tests (K3s)
-    # if: github.event.pull_request.head.repo.full_name == github.repository
+    if: github.event.pull_request.head.repo.full_name == github.repository
     permissions:
       contents: 'read'
       id-token: 'write'
@@ -49,8 +49,7 @@ jobs:
         run: |
           SHORT_SHA="${{ github.sha }}"
           SHORT_SHA="${SHORT_SHA::8}"
-          SALT=$(echo "$RANDOM" | cut -c1-3)
-          echo "RELEASE_NAME=eoapi-$SHORT_SHA-$SALT" >> "$GITHUB_ENV"
+          echo "RELEASE_NAME=eoapi-$SHORT_SHA" >> "$GITHUB_ENV"
 
       - name: Deploy eoAPI
         id: deploy
diff --git a/Makefile b/Makefile
index 1d91c825..1c8f75f1 100755
--- a/Makefile
+++ b/Makefile
@@ -43,7 +43,7 @@ integration:
 
 help:
 	@echo "Makefile commands:"
-	@echo "  make deploy         -  Deploy eoAPI with on connected Kubernetes cluster."
+	@echo "  make deploy         -  Deploy eoAPI to the configured Kubernetes cluster."
 	@echo "  make minikube       -  Install eoAPI on minikube."
 	@echo "  make ingest         -  Ingest STAC collections and items into the database."
 	@echo "  make integration    -  Run integration tests on connected Kubernetes cluster."