From a8a3a782981d78ea531d9faeb1af22bc6e67cafe Mon Sep 17 00:00:00 2001 From: Liam Lloyd-Tucker Date: Thu, 4 Sep 2025 17:18:53 -0700 Subject: [PATCH 1/2] Add Archivematica instances for dev and staging This commit adds terraform definitions for Archivematica instances for our dev and staging environments, as well as Github Actions to deploy those instances. --- .../workflows/deploy_archivematica_dev.yml | 53 ++ .../deploy_archivematica_staging.yml | 53 ++ .gitignore | 1 + .../dev_archivematica_deployment.tf | 812 ++++++++++++++++++ .../test_cluster/dev_gearman_deployment.tf | 60 ++ archivematica/test_cluster/dev_ingress.tf | 132 +++ .../test_cluster/dev_redis_deployment.tf | 85 ++ archivematica/test_cluster/eks-cluster.tf | 54 ++ archivematica/test_cluster/load_balancer.tf | 69 ++ archivematica/test_cluster/locals.tf | 24 + archivematica/test_cluster/main.tf | 41 + archivematica/test_cluster/secrets.tf | 33 + .../staging_archivematica_deployment.tf | 812 ++++++++++++++++++ .../staging_gearman_deployment.tf | 60 ++ archivematica/test_cluster/staging_ingress.tf | 132 +++ .../test_cluster/staging_redis_deployment.tf | 85 ++ archivematica/test_cluster/terraform.tf | 43 + archivematica/test_cluster/variables.tf | 131 +++ 18 files changed, 2680 insertions(+) create mode 100644 .github/workflows/deploy_archivematica_dev.yml create mode 100644 .github/workflows/deploy_archivematica_staging.yml create mode 100644 archivematica/test_cluster/dev_archivematica_deployment.tf create mode 100644 archivematica/test_cluster/dev_gearman_deployment.tf create mode 100644 archivematica/test_cluster/dev_ingress.tf create mode 100644 archivematica/test_cluster/dev_redis_deployment.tf create mode 100644 archivematica/test_cluster/eks-cluster.tf create mode 100644 archivematica/test_cluster/load_balancer.tf create mode 100644 archivematica/test_cluster/locals.tf create mode 100644 archivematica/test_cluster/main.tf create mode 100644 archivematica/test_cluster/secrets.tf create mode 100644 archivematica/test_cluster/staging_archivematica_deployment.tf create mode 100644 archivematica/test_cluster/staging_gearman_deployment.tf create mode 100644 archivematica/test_cluster/staging_ingress.tf create mode 100644 archivematica/test_cluster/staging_redis_deployment.tf create mode 100644 archivematica/test_cluster/terraform.tf create mode 100644 archivematica/test_cluster/variables.tf diff --git a/.github/workflows/deploy_archivematica_dev.yml b/.github/workflows/deploy_archivematica_dev.yml new file mode 100644 index 0000000..6dc62ea --- /dev/null +++ b/.github/workflows/deploy_archivematica_dev.yml @@ -0,0 +1,53 @@ +on: + workflow_dispatch: + +jobs: + build_archivematica: + uses: PermanentOrg/archivematica/.github/workflows/build.yml@main + secrets: inherit + + build_archivematica_storage_service: + uses: PermanentOrg/archivematica-storage-service/.github/workflows/build.yml@main + secrets: inherit + + deploy: + runs-on: ubuntu-latest + needs: + - build_archivematica + - build_archivematica_storage_service + env: + ARCHIVEMATICA_DASHBOARD_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_DASHBOARD_IMAGE_TAG }} + ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG }} + ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG }} + ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG: ${{ needs.build_archivematica_storage_service.outputs.ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG }} + defaults: + run: + working-directory: ./archivematica/test_cluster + steps: + - uses: actions/checkout@v5 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + cli_config_credentials_token: ${{ secrets.TERRAFORM_API_TOKEN }} + - name: Terraform Init + run: terraform init + - name: Terraform Validate + run: terraform validate -no-color + - name: Terraform Plan + run: | + terraform plan -no-color -input=false \ + -var "image_overrides={ + \"archivematica-dashboard-dev\" = \"$ARCHIVEMATICA_DASHBOARD_IMAGE_TAG\", + \"archivematica-mcp-client-dev\" = \"$ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG\", + \"archivematica-mcp-server-dev\" = \"$ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG\", + \"archivematica-storage-service-dev\" = \"$ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG\" + }" + - name: Terraform Apply + run: | + terraform apply -auto-approve -input=false \ + -var "image_overrides={ + \"archivematica-dashboard-dev\" = \"$ARCHIVEMATICA_DASHBOARD_IMAGE_TAG\", + \"archivematica-mcp-client-dev\" = \"$ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG\", + \"archivematica-mcp-server-dev\" = \"$ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG\", + \"archivematica-storage-service-dev\" = \"$ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG\" + }" diff --git a/.github/workflows/deploy_archivematica_staging.yml b/.github/workflows/deploy_archivematica_staging.yml new file mode 100644 index 0000000..652d682 --- /dev/null +++ b/.github/workflows/deploy_archivematica_staging.yml @@ -0,0 +1,53 @@ +on: + workflow_dispatch: + +jobs: + build_archivematica: + uses: PermanentOrg/archivematica/.github/workflows/build.yml@main + secrets: inherit + + build_archivematica_storage_service: + uses: PermanentOrg/archivematica-storage-service/.github/workflows/build.yml@main + secrets: inherit + + deploy: + runs-on: ubuntu-latest + needs: + - build_archivematica + - build_archivematica_storage_service + env: + ARCHIVEMATICA_DASHBOARD_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_DASHBOARD_IMAGE_TAG }} + ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG }} + ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG }} + ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG: ${{ needs.build_archivematica_storage_service.outputs.ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG }} + defaults: + run: + working-directory: ./archivematica/test_cluster + steps: + - uses: actions/checkout@v5 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + cli_config_credentials_token: ${{ secrets.TERRAFORM_API_TOKEN }} + - name: Terraform Init + run: terraform init + - name: Terraform Validate + run: terraform validate -no-color + - name: Terraform Plan + run: | + terraform plan -no-color -input=false \ + -var "image_overrides={ + \"archivematica-dashboard-staging\" = \"$ARCHIVEMATICA_DASHBOARD_IMAGE_TAG\", + \"archivematica-mcp-client-staging\" = \"$ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG\", + \"archivematica-mcp-server-staging\" = \"$ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG\", + \"archivematica-storage-service-staging\" = \"$ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG\", + }" + - name: Terraform Apply + run: | + terraform apply -auto-approve -input=false \ + -var "image_overrides={ + \"archivematica-dashboard-staging\" = \"$ARCHIVEMATICA_DASHBOARD_IMAGE_TAG\", + \"archivematica-mcp-client-staging\" = \"$ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG\", + \"archivematica-mcp-server-staging\" = \"$ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG\", + \"archivematica-storage-service-staging\" = \"$ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG\", + }" diff --git a/.gitignore b/.gitignore index 5ff4697..25f58d3 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,4 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc +.terraform.lock.hcl diff --git a/archivematica/test_cluster/dev_archivematica_deployment.tf b/archivematica/test_cluster/dev_archivematica_deployment.tf new file mode 100644 index 0000000..8114cdd --- /dev/null +++ b/archivematica/test_cluster/dev_archivematica_deployment.tf @@ -0,0 +1,812 @@ +data "kubernetes_resource" "archivematica_dev" { + kind = "Deployment" + api_version = "apps/v1" + metadata { name = "archivematica-dev" } +} + +resource "kubernetes_deployment" "archivematica_dev" { + metadata { + name = "archivematica-dev" + labels = { + App = "archivematica-dev" + Environment = "dev" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-dev" + } + } + template { + metadata { + labels = { + App = "archivematica-dev" + } + } + spec { + security_context { + fs_group = 1000 + fs_group_change_policy = "OnRootMismatch" + } + container { + image = local.desired_images["archivematica-storage-service-dev"] + name = "archivematica-storage-service-dev" + env { + name = "SS_GUNICORN_BIND" + value = "0.0.0.0:8002" + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.production" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + env { + name = "SS_GUNICORN_LOGLEVEL" + value = "debug" + } + env { + name = "SS_GUNICORN_WORKERS" + value = "3" + } + env { + name = "RCLONE_CONFIG" + value = "/var/archivematica/storage_service/.rclone.conf" + } + env { + name = "DJANGO_ALLOWED_HOSTS" + value = "dev.archivematica.permanent.org" + } + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + port { + container_port = 8002 + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "dev-pipeline-data" + } + volume_mount { + mount_path = "/var/archivematica/storage_service" + name = "dev-staging-data" + } + volume_mount { + mount_path = "/home" + name = "dev-location-data" + sub_path = "sips" + } + volume_mount { + mount_path = "/home/transfer" + name = "dev-transfer-share" + } + volume_mount { + mount_path = "/data/storage" + name = "dev-storage-share" + } + } + container { + image = local.desired_images["archivematica-dashboard-dev"] + name = "archivematica-dashboard-dev" + env { + name = "AM_GUNICORN_BIND" + value = "0.0.0.0:8001" + } + env { + name = "DJANGO_SETTINGS_MODULES" + value = "settings.production" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "AM_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "AM_GUNICORN_RELOAD" + value = "true" + } + env { + name = "AM_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "AM_GUNICORN_LOGLEVEL" + value = "debug" + } + env { + name = "AM_GUNICORN_WORKERS" + value = "1" + } + env { + name = "AM_GUNICORN_PROC_NAME" + value = "archivematica-dashboard" + } + env { + name = "AM_GUNICORN_CHDIR" + value = "/src/src/archivematicaCommon/lib/" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PORT" + value = "3306" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_GEARMAN_SERVER" + value = "archivematica-gearman-dev:4730" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_DJANGO_ALLOWED_HOSTS" + value = "dev.archivematica.permanent.org" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_STORAGE_SERVICE_CLIENT_QUICK_TIMEOUT" + value = "20" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + port { + container_port = 8001 + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "dev-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "dev-transfer-share" + } + } + container { + image = local.desired_images["archivematica-mcp-server-dev"] + name = "archivematica-mcp-server-dev" + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.common" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_HOST" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_MCPARCHIVEMATICASERVER" + value = "archivematica-gearman-dev:4730" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_RPC_THREADS" + value = "8" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_WORKER_THREADS" + value = "1" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_CONCURRENT_PACKAGES" + value = "100" + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "333m" + } + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "dev-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "dev-transfer-share" + } + } + init_container { + image = local.desired_images["archivematica-storage-service-dev"] + name = "archivematica-storage-service-migrations" + command = ["sh"] + args = ["-c", "python manage.py migrate --noinput"] + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + } + init_container { + image = local.desired_images["archivematica-storage-service-dev"] + name = "archivematica-storage-service-create-user" + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + env { + name = "AM_SS_USERNAME" + value = "admin" + } + env { + name = "AM_SS_EMAIL" + value = "engineers@permanent.org" + } + env { + name = "AM_SS_PASSWORD" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "AM_SS_PASSWORD" + optional = false + } + } + } + env { + name = "AM_SS_API_KEY" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "AM_SS_API_KEY" + optional = false + } + } + } + command = ["sh"] + args = ["-c", "python manage.py create_user --username=$(AM_SS_USERNAME) --password='$(AM_SS_PASSWORD)' --email=$(AM_SS_EMAIL) --api-key=$(AM_SS_API_KEY) --superuser"] + } + init_container { + image = local.desired_images["archivematica-dashboard-dev"] + name = "archivematica-dashboard-migration" + command = ["sh"] + args = ["-c", "python /src/src/dashboard/src/manage.py migrate --noinput"] + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "AM_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "AM_GUNICORN_RELOAD" + value = "true" + } + env { + name = "AM_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PORT" + value = "3306" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + } + init_container { + image = local.desired_images["archivematica-storage-service-dev"] + name = "archivematica-rclone-configuration" + command = ["sh"] + args = ["-c", "rclone config create permanentb2 b2 account $(BACKBLAZE_KEY_ID) key $(BACKBLAZE_APPLICATION_KEY) --obscure"] + env { + name = "BACKBLAZE_KEY_ID" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "BACKBLAZE_KEY_ID" + optional = false + } + } + } + env { + name = "BACKBLAZE_APPLICATION_KEY" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "BACKBLAZE_APPLICATION_KEY" + optional = false + } + } + } + env { + name = "RCLONE_CONFIG" + value = "/var/archivematica/storage_service/.rclone.conf" + } + volume_mount { + mount_path = "/var/archivematica/storage_service" + name = "dev-staging-data" + } + } + volume { + name = "dev-pipeline-data" + persistent_volume_claim { + claim_name = "dev-pipeline-data" + } + } + volume { + name = "dev-staging-data" + persistent_volume_claim { + claim_name = "dev-staging-data" + } + } + volume { + name = "dev-location-data" + persistent_volume_claim { + claim_name = "dev-location-data" + } + } + volume { + name = "dev-transfer-share" + persistent_volume_claim { + claim_name = "dev-transfer-share" + } + } + volume { + name = "dev-storage-share" + persistent_volume_claim { + claim_name = "dev-storage-share" + } + } + } + } + } +} + +data "kubernetes_resource" "mcp_client_dev" { + kind = "Deployment" + api_version = "apps/v1" + metadata { name = "archivematica-mcp-client-dev" } +} + +resource "kubernetes_deployment" "mcp_client_dev" { + metadata { + name = "archivematica-mcp-client-dev" + labels = { + App = "archivematica-dev" + Environment = "dev" + } + } + spec { + replicas = 4 + selector { + match_labels = { + App = "archivematica-mcp-client-dev" + } + } + template { + metadata { + labels = { + App = "archivematica-mcp-client-dev" + } + } + spec { + container { + image = local.desired_images["archivematica-mcp-client-dev"] + name = "archivematica-mcp-client-dev" + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.common" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_BACKEND" + value = "django.core.mail.backends.smtp.EmailBackend" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_HOST" + value = "smtp.sendgrid.net" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_HOST" + value_from { + secret_key_ref { + name = "dev-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_MCPARCHIVEMATICASERVER" + value = "archivematica-gearman-dev:4730" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_CAPTURE_CLIENT_SCRIPT_OUTPUT" + value = "true" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_STORAGE_SERVICE_CLIENT_QUICK_TIMEOUT" + value = "20" + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "1000m" + } + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "dev-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "dev-transfer-share" + } + } + volume { + name = "dev-pipeline-data" + persistent_volume_claim { + claim_name = "dev-pipeline-data" + } + } + volume { + name = "dev-transfer-share" + persistent_volume_claim { + claim_name = "dev-transfer-share" + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_dashboard_service_dev" { + metadata { + name = "archivematica-dashboard-dev" + } + spec { + type = "ClusterIP" + selector = { + App = "archivematica-dev" + } + port { + port = 8001 + target_port = 8001 + } + } +} + +resource "kubernetes_service" "archivematica_storage_service_dev" { + metadata { + name = "archivematica-storage-dev" + } + spec { + type = "ClusterIP" + selector = { + App = "archivematica-dev" + } + port { + port = 8002 + target_port = 8002 + } + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_dev_pipeline_data_pvc" { + metadata { + name = "dev-pipeline-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_dev_staging_data_pvc" { + metadata { + name = "dev-staging-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_dev_location_data_pvc" { + metadata { + name = "dev-location-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_dev_transfer_share_pvc" { + metadata { + name = "dev-transfer-share" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_dev_storage_share_pvc" { + metadata { + name = "dev-storage-share" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} diff --git a/archivematica/test_cluster/dev_gearman_deployment.tf b/archivematica/test_cluster/dev_gearman_deployment.tf new file mode 100644 index 0000000..68009fd --- /dev/null +++ b/archivematica/test_cluster/dev_gearman_deployment.tf @@ -0,0 +1,60 @@ +resource "kubernetes_deployment" "archivematica_gearman_dev" { + metadata { + name = "archivematica-gearman-dev" + labels = { + App = "archivematica-dev" + Environment = "dev" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-gearman-dev" + } + } + template { + metadata { + labels = { + App = "archivematica-gearman-dev" + } + } + spec { + container { + image = "artefactual/gearmand:1.1.21.4-alpine" + name = "gearman-dev" + args = ["--queue-type=redis", "--redis-server=archivematica-redis-dev", "--redis-port=6379"] + port { + container_port = 4730 + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "1000m" + } + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_gearman_dev" { + metadata { + name = "archivematica-gearman-dev" + } + spec { + selector = { + App = "archivematica-gearman-dev" + } + port { + port = 4730 + target_port = 4730 + } + type = "ClusterIP" + } +} diff --git a/archivematica/test_cluster/dev_ingress.tf b/archivematica/test_cluster/dev_ingress.tf new file mode 100644 index 0000000..5a2423a --- /dev/null +++ b/archivematica/test_cluster/dev_ingress.tf @@ -0,0 +1,132 @@ +resource "kubernetes_ingress_v1" "archivematica_dashboard_ingress_dev" { + metadata { + name = "archivematica-dashboard-ingress-dev" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": 80}, {\"HTTPS\":443}]" + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/group.name" = "archivematica" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.whitelisted_cidrs) + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_dashboard_service_dev.metadata.0.name + port { + number = 8001 + } + } + } + } + } + } + } +} + +resource "kubernetes_ingress_v1" "archivematica_storage_ingress_dev" { + metadata { + name = "archivematica-storage-ingress-dev" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":8000}]" + "alb.ingress.kubernetes.io/group.name" = "archivematica" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.whitelisted_cidrs) + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_storage_service_dev.metadata.0.name + port { + number = 8002 + } + } + } + } + } + } + } +} +resource "kubernetes_ingress_v1" "archivematica_dashboard_internal_ingress_dev" { + metadata { + name = "archivematica-dashboard-internal-ingress-dev" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": 80}, {\"HTTPS\":443}]" + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/group.name" = "archivematica-internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/security-groups" = var.dev_security_group_id + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_dashboard_service_dev.metadata.0.name + port { + number = 8001 + } + } + } + } + } + } + } +} + +resource "kubernetes_ingress_v1" "archivematica_storage_internal_ingress_dev" { + metadata { + name = "archivematica-storage-internal-ingress-dev" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":8000}]" + "alb.ingress.kubernetes.io/group.name" = "archivematica-internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/security-groups" = var.dev_security_group_id + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_storage_service_dev.metadata.0.name + port { + number = 8002 + } + } + } + } + } + } + } +} diff --git a/archivematica/test_cluster/dev_redis_deployment.tf b/archivematica/test_cluster/dev_redis_deployment.tf new file mode 100644 index 0000000..15d0bd1 --- /dev/null +++ b/archivematica/test_cluster/dev_redis_deployment.tf @@ -0,0 +1,85 @@ +resource "kubernetes_deployment" "archivematica_redis_dev" { + metadata { + name = "archivematica-redis-dev" + labels = { + App = "archivematica-dev" + Environment = "dev" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-redis-dev" + } + } + template { + metadata { + labels = { + App = "archivematica-redis-dev" + } + } + spec { + container { + image = "redis:6-alpine" + name = "archivematica-redis-dev" + port { + container_port = 6379 + } + resources { + limits = { + memory = "512Mi" + cpu = "250m" + } + requests = { + memory = "256Mi" + cpu = "250m" + } + } + volume_mount { + name = "archivematica-redis-data-dev" + mount_path = "/data" + } + } + volume { + name = "archivematica-redis-data-dev" + persistent_volume_claim { + claim_name = "archivematica-redis-pvc-dev" + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_redis_dev" { + metadata { + name = "archivematica-redis-dev" + } + spec { + selector = { + App = "archivematica-redis-dev" + } + port { + port = 6379 + target_port = 6379 + } + type = "ClusterIP" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_redis_pvc_dev" { + metadata { + name = "archivematica-redis-pvc-dev" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} diff --git a/archivematica/test_cluster/eks-cluster.tf b/archivematica/test_cluster/eks-cluster.tf new file mode 100644 index 0000000..b0075b3 --- /dev/null +++ b/archivematica/test_cluster/eks-cluster.tf @@ -0,0 +1,54 @@ +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "19.0.4" + + cluster_name = local.cluster_name + cluster_version = "1.32" + + vpc_id = var.vpc_id + subnet_ids = var.subnet_ids + cluster_endpoint_public_access = true + cluster_security_group_id = var.dev_security_group_id + aws_auth_users = [ + { + userarn = "arn:aws:iam::364159549467:user/liam" + username = "liam" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::364159549467:user/cecilia" + username = "cecilia" + groups = ["system:masters"] + } + ] + + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 32 + volume_type = "gp2" + delete_on_termination = true + encrypted = true + } + } + } + } + + eks_managed_node_groups = { + one = { + name = "node-group-1" + + vpc_security_group_ids = [var.dev_security_group_id, var.staging_security_group_id] + + instance_types = ["t3.large"] + + min_size = 3 + max_size = 3 + desired_size = 3 + } + } +} diff --git a/archivematica/test_cluster/load_balancer.tf b/archivematica/test_cluster/load_balancer.tf new file mode 100644 index 0000000..e3f8baa --- /dev/null +++ b/archivematica/test_cluster/load_balancer.tf @@ -0,0 +1,69 @@ +module "lb_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.60.0" + + role_name = "dev_archivematica_lb" + attach_load_balancer_controller_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] + } + } +} + +resource "kubernetes_service_account" "service-account" { + metadata { + name = "aws-load-balancer-controller" + namespace = "kube-system" + labels = { + "app.kubernetes.io/name" = "aws-load-balancer-controller" + "app.kubernetes.io/component" = "controller" + } + annotations = { + "eks.amazonaws.com/role-arn" = module.lb_role.iam_role_arn + "eks.amazonaws.com/sts-regional-endpoints" = "true" + } + } +} + +resource "helm_release" "lb" { + name = "aws-load-balancer-controller" + repository = "https://aws.github.io/eks-charts" + chart = "aws-load-balancer-controller" + namespace = "kube-system" + depends_on = [ + kubernetes_service_account.service-account + ] + + set { + name = "region" + value = var.region + } + + set { + name = "vpcId" + value = var.vpc_id + } + + set { + name = "image.repository" + value = "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon/aws-load-balancer-controller" + } + + set { + name = "serviceAccount.create" + value = "false" + } + + set { + name = "serviceAccount.name" + value = "aws-load-balancer-controller" + } + + set { + name = "clusterName" + value = module.eks.cluster_name + } +} diff --git a/archivematica/test_cluster/locals.tf b/archivematica/test_cluster/locals.tf new file mode 100644 index 0000000..e8cd880 --- /dev/null +++ b/archivematica/test_cluster/locals.tf @@ -0,0 +1,24 @@ +locals { + current_archivematica_dev_deploy = data.kubernetes_resource.archivematica_dev.object + current_mcp_client_dev_deploy = data.kubernetes_resource.mcp_client_dev.object + + current_archivematica_staging_deploy = data.kubernetes_resource.archivematica_staging.object + current_mcp_client_staging_deploy = data.kubernetes_resource.mcp_client_staging.object + + current_containers = concat( + try(local.current_archivematica_dev_deploy.spec.template.spec.containers), + try(local.current_mcp_client_dev_deploy.spec.template.spec.containers), + try(local.current_archivematica_staging_deploy.spec.template.spec.containers), + try(local.current_mcp_client_staging_deploy.spec.template.spec.containers) + ) + + current_images = { for container in local.current_containers : container.name => container.image } + + desired_images = { + for name, image in local.current_images : + name => (contains(keys(var.image_overrides), name) + ? var.image_overrides[name] + : local.current_images[name] + ) + } +} diff --git a/archivematica/test_cluster/main.tf b/archivematica/test_cluster/main.tf new file mode 100644 index 0000000..d015e87 --- /dev/null +++ b/archivematica/test_cluster/main.tf @@ -0,0 +1,41 @@ +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks.cluster_name + ] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + command = "aws" + } + } +} + +provider "aws" { + region = var.region +} + +data "aws_availability_zones" "available" {} + +locals { + cluster_name = "dev-archivematica-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} diff --git a/archivematica/test_cluster/secrets.tf b/archivematica/test_cluster/secrets.tf new file mode 100644 index 0000000..f4990d0 --- /dev/null +++ b/archivematica/test_cluster/secrets.tf @@ -0,0 +1,33 @@ +resource "kubernetes_secret" "dev-archivematica-secrets" { + metadata { + name = "dev-archivematica-secrets" + } + + data = { + "SS_DB_URL" = var.dev_ss_database_url + "AM_SS_PASSWORD" = var.dev_ss_password + "AM_SS_API_KEY" = var.dev_ss_api_key + "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" = var.dev_archivematica_dashboard_db_host + "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" = var.dev_archivematica_dashboard_db_password + "DJANGO_SECRET_KEY" = var.dev_django_secret_key + "BACKBLAZE_KEY_ID" = var.dev_backblaze_key_id + "BACKBLAZE_APPLICATION_KEY" = var.dev_backblaze_application_key + } +} + +resource "kubernetes_secret" "staging-archivematica-secrets" { + metadata { + name = "staging-archivematica-secrets" + } + + data = { + "SS_DB_URL" = var.staging_ss_database_url + "AM_SS_PASSWORD" = var.staging_ss_password + "AM_SS_API_KEY" = var.staging_ss_api_key + "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" = var.staging_archivematica_dashboard_db_host + "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" = var.staging_archivematica_dashboard_db_password + "DJANGO_SECRET_KEY" = var.staging_django_secret_key + "BACKBLAZE_KEY_ID" = var.staging_backblaze_key_id + "BACKBLAZE_APPLICATION_KEY" = var.staging_backblaze_application_key + } +} diff --git a/archivematica/test_cluster/staging_archivematica_deployment.tf b/archivematica/test_cluster/staging_archivematica_deployment.tf new file mode 100644 index 0000000..f65c7f2 --- /dev/null +++ b/archivematica/test_cluster/staging_archivematica_deployment.tf @@ -0,0 +1,812 @@ +data "kubernetes_resource" "archivematica_staging" { + kind = "Deployment" + api_version = "apps/v1" + metadata { name = "archivematica-staging" } +} + +resource "kubernetes_deployment" "archivematica_staging" { + metadata { + name = "archivematica-staging" + labels = { + App = "archivematica-staging" + Environment = "staging" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-staging" + } + } + template { + metadata { + labels = { + App = "archivematica-staging" + } + } + spec { + security_context { + fs_group = 1000 + fs_group_change_policy = "OnRootMismatch" + } + container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:storage-service-main-30826d7" + name = "archivematica-storage-service-staging" + env { + name = "SS_GUNICORN_BIND" + value = "0.0.0.0:8002" + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.production" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + env { + name = "SS_GUNICORN_LOGLEVEL" + value = "debug" + } + env { + name = "SS_GUNICORN_WORKERS" + value = "3" + } + env { + name = "RCLONE_CONFIG" + value = "/var/archivematica/storage_service/.rclone.conf" + } + env { + name = "DJANGO_ALLOWED_HOSTS" + value = "staging.archivematica.permanent.org" + } + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + port { + container_port = 8002 + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "staging-pipeline-data" + } + volume_mount { + mount_path = "/var/archivematica/storage_service" + name = "staging-staging-data" + } + volume_mount { + mount_path = "/home" + name = "staging-location-data" + sub_path = "sips" + } + volume_mount { + mount_path = "/home/transfer" + name = "staging-transfer-share" + } + volume_mount { + mount_path = "/data/storage" + name = "staging-storage-share" + } + } + container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:dashboard-main-0025af3" + name = "archivematica-dashboard-staging" + env { + name = "AM_GUNICORN_BIND" + value = "0.0.0.0:8001" + } + env { + name = "DJANGO_SETTINGS_MODULES" + value = "settings.production" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "AM_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "AM_GUNICORN_RELOAD" + value = "true" + } + env { + name = "AM_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "AM_GUNICORN_LOGLEVEL" + value = "debug" + } + env { + name = "AM_GUNICORN_WORKERS" + value = "1" + } + env { + name = "AM_GUNICORN_PROC_NAME" + value = "archivematica-dashboard" + } + env { + name = "AM_GUNICORN_CHDIR" + value = "/src/src/archivematicaCommon/lib/" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PORT" + value = "3306" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_GEARMAN_SERVER" + value = "archivematica-gearman-staging:4730" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_DJANGO_ALLOWED_HOSTS" + value = "staging.archivematica.permanent.org" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_STORAGE_SERVICE_CLIENT_QUICK_TIMEOUT" + value = "20" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + port { + container_port = 8001 + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "staging-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "staging-transfer-share" + } + } + container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:mcp-server-main-0025af3" + name = "archivematica-mcp-server-staging" + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.common" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_HOST" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_MCPARCHIVEMATICASERVER" + value = "archivematica-gearman-staging:4730" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_RPC_THREADS" + value = "8" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_WORKER_THREADS" + value = "1" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_CONCURRENT_PACKAGES" + value = "100" + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "333m" + } + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "staging-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "staging-transfer-share" + } + } + init_container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:storage-service-main-30826d7" + name = "archivematica-storage-service-migrations" + command = ["sh"] + args = ["-c", "python manage.py migrate --noinput"] + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/staging/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + } + init_container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:storage-service-main-30826d7" + name = "archivematica-storage-service-create-user" + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + env { + name = "AM_SS_USERNAME" + value = "admin" + } + env { + name = "AM_SS_EMAIL" + value = "engineers@permanent.org" + } + env { + name = "AM_SS_PASSWORD" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "AM_SS_PASSWORD" + optional = false + } + } + } + env { + name = "AM_SS_API_KEY" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "AM_SS_API_KEY" + optional = false + } + } + } + command = ["sh"] + args = ["-c", "python manage.py create_user --username=$(AM_SS_USERNAME) --password='$(AM_SS_PASSWORD)' --email=$(AM_SS_EMAIL) --api-key='$(AM_SS_API_KEY)' --superuser"] + } + init_container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:dashboard-main-0025af3" + name = "archivematica-dashboard-migration" + command = ["sh"] + args = ["-c", "python /src/src/dashboard/src/manage.py migrate --noinput"] + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "AM_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "AM_GUNICORN_RELOAD" + value = "true" + } + env { + name = "AM_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PORT" + value = "3306" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + } + init_container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:storage-service-main-30826d7" + name = "archivematica-rclone-configuration" + command = ["sh"] + args = ["-c", "rclone config create permanentb2 b2 account $(BACKBLAZE_KEY_ID) key $(BACKBLAZE_APPLICATION_KEY) --obscure"] + env { + name = "BACKBLAZE_KEY_ID" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "BACKBLAZE_KEY_ID" + optional = false + } + } + } + env { + name = "BACKBLAZE_APPLICATION_KEY" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "BACKBLAZE_APPLICATION_KEY" + optional = false + } + } + } + env { + name = "RCLONE_CONFIG" + value = "/var/archivematica/storage_service/.rclone.conf" + } + volume_mount { + mount_path = "/var/archivematica/storage_service" + name = "staging-staging-data" + } + } + volume { + name = "staging-pipeline-data" + persistent_volume_claim { + claim_name = "staging-pipeline-data" + } + } + volume { + name = "staging-staging-data" + persistent_volume_claim { + claim_name = "staging-staging-data" + } + } + volume { + name = "staging-location-data" + persistent_volume_claim { + claim_name = "staging-location-data" + } + } + volume { + name = "staging-transfer-share" + persistent_volume_claim { + claim_name = "staging-transfer-share" + } + } + volume { + name = "staging-storage-share" + persistent_volume_claim { + claim_name = "staging-storage-share" + } + } + } + } + } +} + +data "kubernetes_resource" "mcp_client_staging" { + kind = "Deployment" + api_version = "apps/v1" + metadata { name = "archivematica-mcp-client-staging" } +} + +resource "kubernetes_deployment" "mcp_client_staging" { + metadata { + name = "archivematica-mcp-client-staging" + labels = { + App = "archivematica-staging" + Environment = "staging" + } + } + spec { + replicas = 4 + selector { + match_labels = { + App = "archivematica-mcp-client-staging" + } + } + template { + metadata { + labels = { + App = "archivematica-mcp-client-staging" + } + } + spec { + container { + image = "364159549467.dkr.ecr.us-west-2.amazonaws.com/archivematica:mcp-client-main-0025af3" + name = "archivematica-mcp-client-staging" + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.common" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_BACKEND" + value = "django.core.mail.backends.smtp.EmailBackend" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_HOST" + value = "smtp.sendgrid.net" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_HOST" + value_from { + secret_key_ref { + name = "staging-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_MCPARCHIVEMATICASERVER" + value = "archivematica-gearman-staging:4730" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_CAPTURE_CLIENT_SCRIPT_OUTPUT" + value = "true" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_STORAGE_SERVICE_CLIENT_QUICK_TIMEOUT" + value = "20" + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "1000m" + } + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "staging-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "staging-transfer-share" + } + } + volume { + name = "staging-pipeline-data" + persistent_volume_claim { + claim_name = "staging-pipeline-data" + } + } + volume { + name = "staging-transfer-share" + persistent_volume_claim { + claim_name = "staging-transfer-share" + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_dashboard_service_staging" { + metadata { + name = "archivematica-dashboard-staging" + } + spec { + type = "ClusterIP" + selector = { + App = "archivematica-staging" + } + port { + port = 8001 + target_port = 8001 + } + } +} + +resource "kubernetes_service" "archivematica_storage_service_staging" { + metadata { + name = "archivematica-storage-staging" + } + spec { + type = "ClusterIP" + selector = { + App = "archivematica-staging" + } + port { + port = 8002 + target_port = 8002 + } + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_staging_pipeline_data_pvc" { + metadata { + name = "staging-pipeline-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_staging_staging_data_pvc" { + metadata { + name = "staging-staging-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_staging_location_data_pvc" { + metadata { + name = "staging-location-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_staging_transfer_share_pvc" { + metadata { + name = "staging-transfer-share" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_staging_storage_share_pvc" { + metadata { + name = "staging-storage-share" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} diff --git a/archivematica/test_cluster/staging_gearman_deployment.tf b/archivematica/test_cluster/staging_gearman_deployment.tf new file mode 100644 index 0000000..207a686 --- /dev/null +++ b/archivematica/test_cluster/staging_gearman_deployment.tf @@ -0,0 +1,60 @@ +resource "kubernetes_deployment" "archivematica_gearman_staging" { + metadata { + name = "archivematica-gearman-staging" + labels = { + App = "archivematica-staging" + Environment = "staging" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-gearman-staging" + } + } + template { + metadata { + labels = { + App = "archivematica-gearman-staging" + } + } + spec { + container { + image = "artefactual/gearmand:1.1.21.4-alpine" + name = "gearman-staging" + args = ["--queue-type=redis", "--redis-server=archivematica-redis-staging", "--redis-port=6379"] + port { + container_port = 4730 + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "1000m" + } + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_gearman_staging" { + metadata { + name = "archivematica-gearman-staging" + } + spec { + selector = { + App = "archivematica-gearman-staging" + } + port { + port = 4730 + target_port = 4730 + } + type = "ClusterIP" + } +} diff --git a/archivematica/test_cluster/staging_ingress.tf b/archivematica/test_cluster/staging_ingress.tf new file mode 100644 index 0000000..bd435d5 --- /dev/null +++ b/archivematica/test_cluster/staging_ingress.tf @@ -0,0 +1,132 @@ +resource "kubernetes_ingress_v1" "archivematica_dashboard_ingress_staging" { + metadata { + name = "archivematica-dashboard-ingress-staging" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": 80}, {\"HTTPS\":443}]" + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/group.name" = "archivematica-staging" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.whitelisted_cidrs) + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_dashboard_service_staging.metadata.0.name + port { + number = 8001 + } + } + } + } + } + } + } +} + +resource "kubernetes_ingress_v1" "archivematica_storage_ingress_staging" { + metadata { + name = "archivematica-storage-ingress-staging" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":8000}]" + "alb.ingress.kubernetes.io/group.name" = "archivematica-staging" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.whitelisted_cidrs) + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_storage_service_staging.metadata.0.name + port { + number = 8002 + } + } + } + } + } + } + } +} +resource "kubernetes_ingress_v1" "archivematica_dashboard_internal_ingress_staging" { + metadata { + name = "archivematica-dashboard-internal-ingress-staging" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": 80}, {\"HTTPS\":443}]" + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/group.name" = "archivematica-staging-internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/security-groups" = var.staging_security_group_id + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_dashboard_service_staging.metadata.0.name + port { + number = 8001 + } + } + } + } + } + } + } +} + +resource "kubernetes_ingress_v1" "archivematica_storage_internal_ingress_staging" { + metadata { + name = "archivematica-storage-internal-ingress-staging" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/e5302df5-6f76-4341-bc41-cd368b6e7411" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":8000}]" + "alb.ingress.kubernetes.io/group.name" = "archivematica-staging-internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/security-groups" = var.staging_security_group_id + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_storage_service_staging.metadata.0.name + port { + number = 8002 + } + } + } + } + } + } + } +} diff --git a/archivematica/test_cluster/staging_redis_deployment.tf b/archivematica/test_cluster/staging_redis_deployment.tf new file mode 100644 index 0000000..8a912ad --- /dev/null +++ b/archivematica/test_cluster/staging_redis_deployment.tf @@ -0,0 +1,85 @@ +resource "kubernetes_deployment" "archivematica_redis_staging" { + metadata { + name = "archivematica-redis-staging" + labels = { + App = "archivematica-staging" + Environment = "staging" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-redis-staging" + } + } + template { + metadata { + labels = { + App = "archivematica-redis-staging" + } + } + spec { + container { + image = "redis:6-alpine" + name = "archivematica-redis-staging" + port { + container_port = 6379 + } + resources { + limits = { + memory = "512Mi" + cpu = "250m" + } + requests = { + memory = "256Mi" + cpu = "250m" + } + } + volume_mount { + name = "archivematica-redis-data-staging" + mount_path = "/data" + } + } + volume { + name = "archivematica-redis-data-staging" + persistent_volume_claim { + claim_name = "archivematica-redis-pvc-staging" + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_redis_staging" { + metadata { + name = "archivematica-redis-staging" + } + spec { + selector = { + App = "archivematica-redis-staging" + } + port { + port = 6379 + target_port = 6379 + } + type = "ClusterIP" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_redis_pvc_staging" { + metadata { + name = "archivematica-redis-pvc-staging" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} diff --git a/archivematica/test_cluster/terraform.tf b/archivematica/test_cluster/terraform.tf new file mode 100644 index 0000000..ff83fb5 --- /dev/null +++ b/archivematica/test_cluster/terraform.tf @@ -0,0 +1,43 @@ +terraform { + cloud { + organization = "PermanentOrg" + + workspaces { + name = "archivematica-dev" + } + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.46.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.4.3" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0.4" + } + + cloudinit = { + source = "hashicorp/cloudinit" + version = "~> 2.2.0" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.16.1" + } + + helm = { + source = "hashicorp/helm" + version = "~> 2.9.0" + } + } + + required_version = "~> 1.3" +} diff --git a/archivematica/test_cluster/variables.tf b/archivematica/test_cluster/variables.tf new file mode 100644 index 0000000..7da9bc9 --- /dev/null +++ b/archivematica/test_cluster/variables.tf @@ -0,0 +1,131 @@ +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} + +variable "dev_env" { + description = "Name of the dev environment" + type = string + default = "dev" +} + +variable "staging_env" { + description = "Name of the staging environment" + type = string + default = "staging" +} + +variable "vpc_id" { + description = "VPC ID" + type = string + default = "vpc-3da37958" +} + +variable "subnet_ids" { + description = "Subnet IDs" + type = list(string) + default = ["subnet-a3f202fa", "subnet-fc843999", "subnet-0fc91a78"] +} +variable "dev_security_group_id" { + description = "ID of the Development security group" + type = string + default = "sg-eca0e789" +} + +variable "staging_security_group_id" { + description = "ID of the Staging security group" + type = string + default = "sg-fea0e79b" +} + +variable "dev_ss_database_url" { + description = "URL of the storage service database for the dev environment" + type = string +} + +variable "staging_ss_database_url" { + description = "URL of the storage service database for the staging environment" + type = string +} + +variable "dev_ss_password" { + description = "Password to the dev environment's storage service" + type = string +} + +variable "staging_ss_password" { + description = "Password to the staging environment's storage service" + type = string +} + +variable "dev_ss_api_key" { + description = "API key for the dev environment's storage service" + type = string +} + +variable "staging_ss_api_key" { + description = "API key for the staging environment's storage service" + type = string +} + +variable "dev_archivematica_dashboard_db_host" { + description = "Host address of the dev environment's database" + type = string +} + +variable "staging_archivematica_dashboard_db_host" { + description = "Host address of the staging environment's database" + type = string +} + +variable "dev_archivematica_dashboard_db_password" { + description = "Password to the dev environment's database" + type = string +} + +variable "staging_archivematica_dashboard_db_password" { + description = "Password to the staging environment's database" + type = string +} + +variable "dev_django_secret_key" { + description = "Signing key for django in the dev environment" + type = string +} + +variable "staging_django_secret_key" { + description = "Signing key for django in the staging environment" + type = string +} + +variable "dev_backblaze_key_id" { + description = "Key ID for Backblaze in the dev environment" + type = string +} + +variable "staging_backblaze_key_id" { + description = "Key ID for Backblaze in the staging environment" + type = string +} + +variable "dev_backblaze_application_key" { + description = "Application key for Backblaze in the dev environment" + type = string +} + +variable "staging_backblaze_application_key" { + description = "Application key for Backblaze in the staging environment" + type = string +} + +variable "whitelisted_cidrs" { + description = "IPs allowed to access Archivematica from outside the security group" + type = list(string) +} + +variable "image_overrides" { + description = "A map of docker images to be updated" + type = map(string) + default = {} +} From bc0a032f71bbeb353a762039fca0aac57b8be40f Mon Sep 17 00:00:00 2001 From: Liam Lloyd-Tucker Date: Tue, 23 Sep 2025 17:21:20 -0700 Subject: [PATCH 2/2] Add production deployment of Archivematica --- .../workflows/deploy_archivematica_prod.yml | 49 ++ .../deploy_archivematica_staging.yml | 15 + .../prod_cluster/archivematica_deployment.tf | 812 ++++++++++++++++++ archivematica/prod_cluster/eks-cluster.tf | 54 ++ .../prod_cluster/gearman_deployment.tf | 60 ++ archivematica/prod_cluster/ingress.tf | 132 +++ archivematica/prod_cluster/load_balancer.tf | 69 ++ archivematica/prod_cluster/locals.tf | 19 + archivematica/prod_cluster/main.tf | 41 + .../prod_cluster/redis_deployment.tf | 85 ++ archivematica/prod_cluster/secrets.tf | 16 + archivematica/prod_cluster/terraform.tf | 43 + archivematica/prod_cluster/variables.tf | 79 ++ 13 files changed, 1474 insertions(+) create mode 100644 .github/workflows/deploy_archivematica_prod.yml create mode 100644 archivematica/prod_cluster/archivematica_deployment.tf create mode 100644 archivematica/prod_cluster/eks-cluster.tf create mode 100644 archivematica/prod_cluster/gearman_deployment.tf create mode 100644 archivematica/prod_cluster/ingress.tf create mode 100644 archivematica/prod_cluster/load_balancer.tf create mode 100644 archivematica/prod_cluster/locals.tf create mode 100644 archivematica/prod_cluster/main.tf create mode 100644 archivematica/prod_cluster/redis_deployment.tf create mode 100644 archivematica/prod_cluster/secrets.tf create mode 100644 archivematica/prod_cluster/terraform.tf create mode 100644 archivematica/prod_cluster/variables.tf diff --git a/.github/workflows/deploy_archivematica_prod.yml b/.github/workflows/deploy_archivematica_prod.yml new file mode 100644 index 0000000..2b9cd15 --- /dev/null +++ b/.github/workflows/deploy_archivematica_prod.yml @@ -0,0 +1,49 @@ +on: + workflow_dispatch: + +jobs: + deploy_archivematica_staging: + uses: ./.github/workflows/deploy_archivematica_staging.yml + secrets: inherit + + deploy: + runs-on: ubuntu-latest + needs: + - deploy_archivematica_staging + environment: prod + env: + ARCHIVEMATICA_DASHBOARD_IMAGE_TAG: ${{ needs.deploy_archivematica_staging.outputs.ARCHIVEMATICA_DASHBOARD_IMAGE_TAG }} + ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG: ${{ needs.deploy_archivematica_staging.outputs.ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG }} + ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG: ${{ needs.deploy_archivematica_staging.outputs.ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG }} + ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG: ${{ needs.deploy_archivematica_staging.outputs.ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG }} + defaults: + run: + working-directory: ./archivematica/prod_cluster + steps: + - uses: actions/checkout@v5 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + with: + cli_config_credentials_token: ${{ secrets.TERRAFORM_API_TOKEN }} + - name: Terraform Init + run: terraform init + - name: Terraform Validate + run: terraform validate -no-color + - name: Terraform Plan + run: | + terraform plan -no-color -input=false \ + -var "image_overrides={ + \"archivematica-dashboard-prod\" = \"$ARCHIVEMATICA_DASHBOARD_IMAGE_TAG\", + \"archivematica-mcp-client-prod\" = \"$ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG\", + \"archivematica-mcp-server-prod\" = \"$ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG\", + \"archivematica-storage-service-prod\" = \"$ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG\", + }" + - name: Terraform Apply + run: | + terraform apply -auto-approve -input=false \ + -var "image_overrides={ + \"archivematica-dashboard-prod\" = \"$ARCHIVEMATICA_DASHBOARD_IMAGE_TAG\", + \"archivematica-mcp-client-prod\" = \"$ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG\", + \"archivematica-mcp-server-prod\" = \"$ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG\", + \"archivematica-storage-service-prod\" = \"$ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG\", + }" diff --git a/.github/workflows/deploy_archivematica_staging.yml b/.github/workflows/deploy_archivematica_staging.yml index 652d682..f656204 100644 --- a/.github/workflows/deploy_archivematica_staging.yml +++ b/.github/workflows/deploy_archivematica_staging.yml @@ -1,5 +1,15 @@ on: workflow_dispatch: + workflow_call: + outputs: + ARCHIVEMATICA_DASHBOARD_IMAGE_TAG: + value: ${{ jobs.deploy.outputs.ARCHIVEMATICA_DASHBOARD_IMAGE_TAG }} + ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG: + value: ${{ jobs.deploy.outputs.ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG }} + ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG: + value: ${{ jobs.deploy.outputs.ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG }} + ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG: + value: ${{ jobs.deploy.outputs.ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG }} jobs: build_archivematica: @@ -12,6 +22,11 @@ jobs: deploy: runs-on: ubuntu-latest + outputs: + ARCHIVEMATICA_DASHBOARD_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_DASHBOARD_IMAGE_TAG }} + ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_MCP_SERVER_IMAGE_TAG }} + ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG: ${{ needs.build_archivematica.outputs.ARCHIVEMATICA_MCP_CLIENT_IMAGE_TAG }} + ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG: ${{ needs.build_archivematica_storage_service.outputs.ARCHIVEMATICA_STORAGE_SERVICE_IMAGE_TAG }} needs: - build_archivematica - build_archivematica_storage_service diff --git a/archivematica/prod_cluster/archivematica_deployment.tf b/archivematica/prod_cluster/archivematica_deployment.tf new file mode 100644 index 0000000..1229db0 --- /dev/null +++ b/archivematica/prod_cluster/archivematica_deployment.tf @@ -0,0 +1,812 @@ +data "kubernetes_resource" "archivematica_prod" { + kind = "Deployment" + api_version = "apps/v1" + metadata { name = "archivematica-prod" } +} + +resource "kubernetes_deployment" "archivematica_prod" { + metadata { + name = "archivematica-prod" + labels = { + App = "archivematica-prod" + Environment = "prod" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-prod" + } + } + template { + metadata { + labels = { + App = "archivematica-prod" + } + } + spec { + security_context { + fs_group = 1000 + fs_group_change_policy = "OnRootMismatch" + } + container { + image = local.desired_images["archivematica-storage-service-prod"] + name = "archivematica-storage-service-prod" + env { + name = "SS_GUNICORN_BIND" + value = "0.0.0.0:8002" + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.production" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + env { + name = "SS_GUNICORN_LOGLEVEL" + value = "debug" + } + env { + name = "SS_GUNICORN_WORKERS" + value = "3" + } + env { + name = "RCLONE_CONFIG" + value = "/var/archivematica/storage_service/.rclone.conf" + } + env { + name = "DJANGO_ALLOWED_HOSTS" + value = "archivematica.permanent.org" + } + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + port { + container_port = 8002 + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "prod-pipeline-data" + } + volume_mount { + mount_path = "/var/archivematica/storage_service" + name = "prod-staging-data" + } + volume_mount { + mount_path = "/home" + name = "prod-location-data" + sub_path = "sips" + } + volume_mount { + mount_path = "/home/transfer" + name = "prod-transfer-share" + } + volume_mount { + mount_path = "/data/storage" + name = "prod-storage-share" + } + } + container { + image = local.desired_images["archivematica-dashboard-prod"] + name = "archivematica-dashboard-prod" + env { + name = "AM_GUNICORN_BIND" + value = "0.0.0.0:8001" + } + env { + name = "DJANGO_SETTINGS_MODULES" + value = "settings.production" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "AM_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "AM_GUNICORN_RELOAD" + value = "true" + } + env { + name = "AM_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "AM_GUNICORN_LOGLEVEL" + value = "debug" + } + env { + name = "AM_GUNICORN_WORKERS" + value = "1" + } + env { + name = "AM_GUNICORN_PROC_NAME" + value = "archivematica-dashboard" + } + env { + name = "AM_GUNICORN_CHDIR" + value = "/src/src/archivematicaCommon/lib/" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PORT" + value = "3306" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_GEARMAN_SERVER" + value = "archivematica-gearman-prod:4730" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_DJANGO_ALLOWED_HOSTS" + value = "archivematica.permanent.org" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_STORAGE_SERVICE_CLIENT_QUICK_TIMEOUT" + value = "20" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_DASHBOARD_DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + port { + container_port = 8001 + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "prod-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "prod-transfer-share" + } + } + container { + image = local.desired_images["archivematica-mcp-server-prod"] + name = "archivematica-mcp-server-prod" + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.common" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_HOST" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPSERVER_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_MCPARCHIVEMATICASERVER" + value = "archivematica-gearman-prod:4730" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_RPC_THREADS" + value = "8" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_WORKER_THREADS" + value = "1" + } + env { + name = "ARCHIVEMATICA_MCPSERVER_MCPSERVER_CONCURRENT_PACKAGES" + value = "100" + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "333m" + } + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "prod-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "prod-transfer-share" + } + } + init_container { + image = local.desired_images["archivematica-storage-service-prod"] + name = "archivematica-storage-service-migrations" + command = ["sh"] + args = ["-c", "python manage.py migrate --noinput"] + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + } + init_container { + image = local.desired_images["archivematica-storage-service-prod"] + name = "archivematica-storage-service-create-user" + env { + name = "DJANGO_SETTINGS_MODULE" + value = "storage_service.settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "SS_GNUPG_HOME_PATH" + value = "/var/archivematica/storage_service/.gnupg" + } + env { + name = "SS_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "SS_GUNICORN_RELOAD" + value = "true" + } + env { + name = "SS_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "SS_DB_URL" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "SS_DB_URL" + optional = false + } + } + } + env { + name = "AM_SS_USERNAME" + value = "admin" + } + env { + name = "AM_SS_EMAIL" + value = "engineers@permanent.org" + } + env { + name = "AM_SS_PASSWORD" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "AM_SS_PASSWORD" + optional = false + } + } + } + env { + name = "AM_SS_API_KEY" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "AM_SS_API_KEY" + optional = false + } + } + } + command = ["sh"] + args = ["-c", "python manage.py create_user --username=$(AM_SS_USERNAME) --password='$(AM_SS_PASSWORD)' --email=$(AM_SS_EMAIL) --api-key=$(AM_SS_API_KEY) --superuser"] + } + init_container { + image = local.desired_images["archivematica-dashboard-prod"] + name = "archivematica-dashboard-migration" + command = ["sh"] + args = ["-c", "python /src/src/dashboard/src/manage.py migrate --noinput"] + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.local" + } + env { + name = "FORWARDED_ALLOW_IPS" + value = "*" + } + env { + name = "AM_GUNICORN_ACCESSLOG" + value = "/dev/null" + } + env { + name = "AM_GUNICORN_RELOAD" + value = "true" + } + env { + name = "AM_GUNICORN_RELOAD_ENGINE" + value = "auto" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PORT" + value = "3306" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + } + init_container { + image = local.desired_images["archivematica-storage-service-prod"] + name = "archivematica-rclone-configuration" + command = ["sh"] + args = ["-c", "rclone config create permanentb2 b2 account $(BACKBLAZE_KEY_ID) key $(BACKBLAZE_APPLICATION_KEY) --obscure"] + env { + name = "BACKBLAZE_KEY_ID" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "BACKBLAZE_KEY_ID" + optional = false + } + } + } + env { + name = "BACKBLAZE_APPLICATION_KEY" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "BACKBLAZE_APPLICATION_KEY" + optional = false + } + } + } + env { + name = "RCLONE_CONFIG" + value = "/var/archivematica/storage_service/.rclone.conf" + } + volume_mount { + mount_path = "/var/archivematica/storage_service" + name = "prod-staging-data" + } + } + volume { + name = "prod-pipeline-data" + persistent_volume_claim { + claim_name = "prod-pipeline-data" + } + } + volume { + name = "prod-staging-data" + persistent_volume_claim { + claim_name = "prod-staging-data" + } + } + volume { + name = "prod-location-data" + persistent_volume_claim { + claim_name = "prod-location-data" + } + } + volume { + name = "prod-transfer-share" + persistent_volume_claim { + claim_name = "prod-transfer-share" + } + } + volume { + name = "prod-storage-share" + persistent_volume_claim { + claim_name = "prod-storage-share" + } + } + } + } + } +} + +data "kubernetes_resource" "mcp_client_prod" { + kind = "Deployment" + api_version = "apps/v1" + metadata { name = "archivematica-mcp-client-prod" } +} + +resource "kubernetes_deployment" "mcp_client_prod" { + metadata { + name = "archivematica-mcp-client-prod" + labels = { + App = "archivematica-prod" + Environment = "prod" + } + } + spec { + replicas = 4 + selector { + match_labels = { + App = "archivematica-mcp-client-prod" + } + } + template { + metadata { + labels = { + App = "archivematica-mcp-client-prod" + } + } + spec { + container { + image = local.desired_images["archivematica-mcp-client-prod"] + name = "archivematica-mcp-client-prod" + env { + name = "DJANGO_SECRET_KEY" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "DJANGO_SECRET_KEY" + optional = false + } + } + } + env { + name = "DJANGO_SETTINGS_MODULE" + value = "settings.common" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_BACKEND" + value = "django.core.mail.backends.smtp.EmailBackend" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_HOST" + value = "smtp.sendgrid.net" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_EMAIL_PORT" + value = "587" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_USER" + value = "archivematica" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_PASSWORD" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_HOST" + value_from { + secret_key_ref { + name = "prod-archivematica-secrets" + key = "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" + optional = false + } + } + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_CLIENT_DATABASE" + value = "MCP" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_MCPARCHIVEMATICASERVER" + value = "archivematica-gearman-prod:4730" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_SEARCH_ENABLED" + value = "false" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_CAPTURE_CLIENT_SCRIPT_OUTPUT" + value = "true" + } + env { + name = "ARCHIVEMATICA_MCPCLIENT_MCPCLIENT_STORAGE_SERVICE_CLIENT_QUICK_TIMEOUT" + value = "20" + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "1000m" + } + } + volume_mount { + mount_path = "/var/archivematica/sharedDirectory" + name = "prod-pipeline-data" + } + volume_mount { + mount_path = "/home/transfer" + name = "prod-transfer-share" + } + } + volume { + name = "prod-pipeline-data" + persistent_volume_claim { + claim_name = "prod-pipeline-data" + } + } + volume { + name = "prod-transfer-share" + persistent_volume_claim { + claim_name = "prod-transfer-share" + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_dashboard_service_prod" { + metadata { + name = "archivematica-dashboard-prod" + } + spec { + type = "ClusterIP" + selector = { + App = "archivematica-prod" + } + port { + port = 8001 + target_port = 8001 + } + } +} + +resource "kubernetes_service" "archivematica_storage_service_prod" { + metadata { + name = "archivematica-storage-prod" + } + spec { + type = "ClusterIP" + selector = { + App = "archivematica-prod" + } + port { + port = 8002 + target_port = 8002 + } + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_prod_pipeline_data_pvc" { + metadata { + name = "prod-pipeline-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "64Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_prod_staging_data_pvc" { + metadata { + name = "prod-staging-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "64Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_prod_location_data_pvc" { + metadata { + name = "prod-location-data" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "64Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_prod_transfer_share_pvc" { + metadata { + name = "prod-transfer-share" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "64Gi" + } + } + + storage_class_name = "gp2" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_prod_storage_share_pvc" { + metadata { + name = "prod-storage-share" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "64Gi" + } + } + + storage_class_name = "gp2" + } +} diff --git a/archivematica/prod_cluster/eks-cluster.tf b/archivematica/prod_cluster/eks-cluster.tf new file mode 100644 index 0000000..bf29329 --- /dev/null +++ b/archivematica/prod_cluster/eks-cluster.tf @@ -0,0 +1,54 @@ +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "19.0.4" + + cluster_name = local.cluster_name + cluster_version = "1.32" + + vpc_id = var.vpc_id + subnet_ids = var.subnet_ids + cluster_endpoint_public_access = true + cluster_security_group_id = var.security_group_id + aws_auth_users = [ + { + userarn = "arn:aws:iam::364159549467:user/liam" + username = "liam" + groups = ["system:masters"] + }, + { + userarn = "arn:aws:iam::364159549467:user/cecilia" + username = "cecilia" + groups = ["system:masters"] + } + ] + + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = 32 + volume_type = "gp2" + delete_on_termination = true + encrypted = true + } + } + } + } + + eks_managed_node_groups = { + one = { + name = "node-group-1" + + vpc_security_group_ids = [var.security_group_id] + + instance_types = ["t3.large"] + + min_size = 3 + max_size = 3 + desired_size = 3 + } + } +} diff --git a/archivematica/prod_cluster/gearman_deployment.tf b/archivematica/prod_cluster/gearman_deployment.tf new file mode 100644 index 0000000..49087d2 --- /dev/null +++ b/archivematica/prod_cluster/gearman_deployment.tf @@ -0,0 +1,60 @@ +resource "kubernetes_deployment" "archivematica_gearman_prod" { + metadata { + name = "archivematica-gearman-prod" + labels = { + App = "archivematica-prod" + Environment = "prod" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-gearman-prod" + } + } + template { + metadata { + labels = { + App = "archivematica-gearman-prod" + } + } + spec { + container { + image = "artefactual/gearmand:1.1.21.4-alpine" + name = "gearman-prod" + args = ["--queue-type=redis", "--redis-server=archivematica-redis-prod", "--redis-port=6379"] + port { + container_port = 4730 + } + resources { + requests = { + memory = "256Mi" + cpu = "1m" + } + limits = { + memory = "2048Mi" + cpu = "1000m" + } + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_gearman_prod" { + metadata { + name = "archivematica-gearman-prod" + } + spec { + selector = { + App = "archivematica-gearman-prod" + } + port { + port = 4730 + target_port = 4730 + } + type = "ClusterIP" + } +} diff --git a/archivematica/prod_cluster/ingress.tf b/archivematica/prod_cluster/ingress.tf new file mode 100644 index 0000000..bba7175 --- /dev/null +++ b/archivematica/prod_cluster/ingress.tf @@ -0,0 +1,132 @@ +resource "kubernetes_ingress_v1" "archivematica_dashboard_ingress_prod" { + metadata { + name = "archivematica-dashboard-ingress-dev" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/ba6adc44-5a9d-47cb-a64f-ddb840f6f19d" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": 80}, {\"HTTPS\":443}]" + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/group.name" = "archivematica" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.whitelisted_cidrs) + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_dashboard_service_prod.metadata.0.name + port { + number = 8001 + } + } + } + } + } + } + } +} + +resource "kubernetes_ingress_v1" "archivematica_storage_ingress_prod" { + metadata { + name = "archivematica-storage-ingress-prod" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internet-facing" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/ba6adc44-5a9d-47cb-a64f-ddb840f6f19d" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":8000}]" + "alb.ingress.kubernetes.io/group.name" = "archivematica" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/inbound-cidrs" = join(",", var.whitelisted_cidrs) + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_storage_service_prod.metadata.0.name + port { + number = 8002 + } + } + } + } + } + } + } +} +resource "kubernetes_ingress_v1" "archivematica_dashboard_internal_ingress_prod" { + metadata { + name = "archivematica-dashboard-internal-ingress-prod" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/ba6adc44-5a9d-47cb-a64f-ddb840f6f19d" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTP\": 80}, {\"HTTPS\":443}]" + "alb.ingress.kubernetes.io/ssl-redirect" = "443" + "alb.ingress.kubernetes.io/group.name" = "archivematica-internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/security-groups" = var.security_group_id + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_dashboard_service_prod.metadata.0.name + port { + number = 8001 + } + } + } + } + } + } + } +} + +resource "kubernetes_ingress_v1" "archivematica_storage_internal_ingress_prod" { + metadata { + name = "archivematica-storage-internal-ingress-prod" + annotations = { + "alb.ingress.kubernetes.io/scheme" = "internal" + "alb.ingress.kubernetes.io/subnets" = "${var.subnet_ids[0]},${var.subnet_ids[1]},${var.subnet_ids[2]}" + "alb.ingress.kubernetes.io/certificate-arn" = "arn:aws:acm:us-west-2:364159549467:certificate/ba6adc44-5a9d-47cb-a64f-ddb840f6f19d" + "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\":8000}]" + "alb.ingress.kubernetes.io/group.name" = "archivematica-internal" + "alb.ingress.kubernetes.io/target-type" = "ip" + "alb.ingress.kubernetes.io/security-groups" = var.security_group_id + } + } + spec { + ingress_class_name = "alb" + rule { + http { + path { + path = "/*" + backend { + service { + name = kubernetes_service.archivematica_storage_service_prod.metadata.0.name + port { + number = 8002 + } + } + } + } + } + } + } +} diff --git a/archivematica/prod_cluster/load_balancer.tf b/archivematica/prod_cluster/load_balancer.tf new file mode 100644 index 0000000..e45be8b --- /dev/null +++ b/archivematica/prod_cluster/load_balancer.tf @@ -0,0 +1,69 @@ +module "lb_role" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "5.60.0" + + role_name = "prod_archivematica_lb" + attach_load_balancer_controller_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-load-balancer-controller"] + } + } +} + +resource "kubernetes_service_account" "service-account" { + metadata { + name = "aws-load-balancer-controller" + namespace = "kube-system" + labels = { + "app.kubernetes.io/name" = "aws-load-balancer-controller" + "app.kubernetes.io/component" = "controller" + } + annotations = { + "eks.amazonaws.com/role-arn" = module.lb_role.iam_role_arn + "eks.amazonaws.com/sts-regional-endpoints" = "true" + } + } +} + +resource "helm_release" "lb" { + name = "aws-load-balancer-controller" + repository = "https://aws.github.io/eks-charts" + chart = "aws-load-balancer-controller" + namespace = "kube-system" + depends_on = [ + kubernetes_service_account.service-account + ] + + set { + name = "region" + value = var.region + } + + set { + name = "vpcId" + value = var.vpc_id + } + + set { + name = "image.repository" + value = "602401143452.dkr.ecr.us-west-2.amazonaws.com/amazon/aws-load-balancer-controller" + } + + set { + name = "serviceAccount.create" + value = "false" + } + + set { + name = "serviceAccount.name" + value = "aws-load-balancer-controller" + } + + set { + name = "clusterName" + value = module.eks.cluster_name + } +} diff --git a/archivematica/prod_cluster/locals.tf b/archivematica/prod_cluster/locals.tf new file mode 100644 index 0000000..263b7ff --- /dev/null +++ b/archivematica/prod_cluster/locals.tf @@ -0,0 +1,19 @@ +locals { + current_archivematica_prod_deploy = data.kubernetes_resource.archivematica_prod.object + current_mcp_client_prod_deploy = data.kubernetes_resource.mcp_client_prod.object + + current_containers = concat( + try(local.current_archivematica_prod_deploy.spec.template.spec.containers), + try(local.current_mcp_client_prod_deploy.spec.template.spec.containers) + ) + + current_images = { for container in local.current_containers : container.name => container.image } + + desired_images = { + for name, image in local.current_images : + name => (contains(keys(var.image_overrides), name) + ? var.image_overrides[name] + : local.current_images[name] + ) + } +} diff --git a/archivematica/prod_cluster/main.tf b/archivematica/prod_cluster/main.tf new file mode 100644 index 0000000..58e29a3 --- /dev/null +++ b/archivematica/prod_cluster/main.tf @@ -0,0 +1,41 @@ +provider "kubernetes" { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks.cluster_name + ] + } +} + +provider "helm" { + kubernetes { + host = module.eks.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1" + args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name] + command = "aws" + } + } +} + +provider "aws" { + region = var.region +} + +data "aws_availability_zones" "available" {} + +locals { + cluster_name = "prod-archivematica-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} diff --git a/archivematica/prod_cluster/redis_deployment.tf b/archivematica/prod_cluster/redis_deployment.tf new file mode 100644 index 0000000..666aa1a --- /dev/null +++ b/archivematica/prod_cluster/redis_deployment.tf @@ -0,0 +1,85 @@ +resource "kubernetes_deployment" "archivematica_redis_prod" { + metadata { + name = "archivematica-redis-prod" + labels = { + App = "archivematica-prod" + Environment = "prod" + } + } + spec { + replicas = 1 + selector { + match_labels = { + App = "archivematica-redis-prod" + } + } + template { + metadata { + labels = { + App = "archivematica-redis-prod" + } + } + spec { + container { + image = "redis:6-alpine" + name = "archivematica-redis-prod" + port { + container_port = 6379 + } + resources { + limits = { + memory = "512Mi" + cpu = "250m" + } + requests = { + memory = "256Mi" + cpu = "250m" + } + } + volume_mount { + name = "archivematica-redis-data-prod" + mount_path = "/data" + } + } + volume { + name = "archivematica-redis-data-prod" + persistent_volume_claim { + claim_name = "archivematica-redis-pvc-prod" + } + } + } + } + } +} + +resource "kubernetes_service" "archivematica_redis_prod" { + metadata { + name = "archivematica-redis-prod" + } + spec { + selector = { + App = "archivematica-redis-prod" + } + port { + port = 6379 + target_port = 6379 + } + type = "ClusterIP" + } +} + +resource "kubernetes_persistent_volume_claim" "archivematica_redis_pvc_prod" { + metadata { + name = "archivematica-redis-pvc-prod" + } + spec { + access_modes = ["ReadWriteOnce"] + resources { + requests = { + storage = "2Gi" + } + } + + storage_class_name = "gp2" + } +} diff --git a/archivematica/prod_cluster/secrets.tf b/archivematica/prod_cluster/secrets.tf new file mode 100644 index 0000000..2696ef0 --- /dev/null +++ b/archivematica/prod_cluster/secrets.tf @@ -0,0 +1,16 @@ +resource "kubernetes_secret" "prod-archivematica-secrets" { + metadata { + name = "prod-archivematica-secrets" + } + + data = { + "SS_DB_URL" = var.prod_ss_database_url + "AM_SS_PASSWORD" = var.prod_ss_password + "AM_SS_API_KEY" = var.prod_ss_api_key + "ARCHIVEMATICA_DASHBOARD_CLIENT_HOST" = var.prod_archivematica_dashboard_db_host + "ARCHIVEMATICA_DASHBOARD_CLIENT_PASSWORD" = var.prod_archivematica_dashboard_db_password + "DJANGO_SECRET_KEY" = var.prod_django_secret_key + "BACKBLAZE_KEY_ID" = var.prod_backblaze_key_id + "BACKBLAZE_APPLICATION_KEY" = var.prod_backblaze_application_key + } +} diff --git a/archivematica/prod_cluster/terraform.tf b/archivematica/prod_cluster/terraform.tf new file mode 100644 index 0000000..eb1fc5f --- /dev/null +++ b/archivematica/prod_cluster/terraform.tf @@ -0,0 +1,43 @@ +terraform { + cloud { + organization = "PermanentOrg" + + workspaces { + name = "archivematica-prod" + } + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.46.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.4.3" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0.4" + } + + cloudinit = { + source = "hashicorp/cloudinit" + version = "~> 2.2.0" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = "~> 2.16.1" + } + + helm = { + source = "hashicorp/helm" + version = "~> 2.9.0" + } + } + + required_version = "~> 1.3" +} diff --git a/archivematica/prod_cluster/variables.tf b/archivematica/prod_cluster/variables.tf new file mode 100644 index 0000000..bb0b4b8 --- /dev/null +++ b/archivematica/prod_cluster/variables.tf @@ -0,0 +1,79 @@ +variable "region" { + description = "AWS region" + type = string + default = "us-west-2" +} + +variable "prod_env" { + description = "Name of the prod environment" + type = string + default = "prod" +} + +variable "vpc_id" { + description = "VPC ID" + type = string + default = "vpc-3da37958" +} + +variable "subnet_ids" { + description = "Subnet IDs" + type = list(string) + default = ["subnet-a3f202fa", "subnet-fc843999", "subnet-0fc91a78"] +} +variable "security_group_id" { + description = "ID of the prodelopment security group" + type = string + default = "sg-9c3f62f9" +} + +variable "prod_ss_database_url" { + description = "URL of the storage service database for the prod environment" + type = string +} + +variable "prod_ss_password" { + description = "Password to the prod environment's storage service" + type = string +} + +variable "prod_ss_api_key" { + description = "API key for the prod environment's storage service" + type = string +} + +variable "prod_archivematica_dashboard_db_host" { + description = "Host address of the prod environment's database" + type = string +} + +variable "prod_archivematica_dashboard_db_password" { + description = "Password to the prod environment's database" + type = string +} + +variable "prod_django_secret_key" { + description = "Signing key for django in the prod environment" + type = string +} + +variable "prod_backblaze_key_id" { + description = "Key ID for Backblaze in the prod environment" + type = string +} + +variable "prod_backblaze_application_key" { + description = "Application key for Backblaze in the prod environment" + type = string +} + +variable "whitelisted_cidrs" { + description = "IPs allowed to access Archivematica from outside the security group" + type = list(string) +} + +variable "image_overrides" { + description = "A map of docker images to be updated" + type = map(string) + default = {} +}