chore(conf): migrate production static storage to GCS buckets

Migrate the production static storage from the local filesystem to a Google Cloud Storage bucket. Other changes included are:

- Change the deployment process to also perform database migrations and collect static files. The prior implementation would perform this during container startup which would in turn lengthen the container startup by a few more seconds.
- Change environment loading behaviour in production to first load the environment variables from a `.env` file if a path to one was provided. If not, load the environment variables from the GCP Secret Manager service. If neither of these is successful, fail with a descriptive error message.

Included also are other minor improvements and fixes.

Dockerfile

@@ -62,12 +62,15 @@ COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
 RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
   && rm -rf /wheels/
 
-
-COPY ./entrypoint /entrypoint
-RUN sed -i 's/\r$//g' /entrypoint
-RUN chmod +x /entrypoint
-
 # Copy application code to WORKDIR
 COPY --from=client-builder ${APP_HOME} ${APP_HOME}
 
-ENTRYPOINT ["/entrypoint"]
+
+# Run the web service on container startup.
+# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud
+# Run to handle instance scaling.
+CMD gunicorn config.asgi \
+    --bind 0.0.0.0:$PORT \
+    --timeout 0 \
+    --chdir=/app \
+    -k uvicorn.workers.UvicornWorker

cloudbuild.yaml

@@ -1,6 +1,7 @@
 steps:
-  # build the container image
-  - name: "gcr.io/cloud-builders/docker"
+  # Build the container image
+  - id: "build image"
+    name: "gcr.io/cloud-builders/docker"
     args:
       [
         "build",
@@ -10,12 +11,69 @@ steps:
       ]
 
   # Push the container image to Container Registry
-  - name: "gcr.io/cloud-builders/docker"
+  - id: "push image"
+    name: "gcr.io/cloud-builders/docker"
     args:
-      ["push", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA"]
+      [
+        "push",
+        "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA"
+      ]
+
+  # Apply the latest migrations
+  - id: "apply migrations"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/cqi-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.prod",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "migrate",
+      ]
+
+  # Create cache table
+  - id: "create cache table"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/cqi-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.prod",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "createcachetable",
+      ]
+
+  # Collect static files
+  - id: "collect static files"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/cqi-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.prod",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "collectstatic", "--noinput"
+      ]
+
+  # Collect static files
+  - id: "compress static assets"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/cqi-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.prod",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "compress"
+      ]
 
   # Deploy an image from Container Registry to Cloud Run
-  - name: "gcr.io/cloud-builders/gcloud"
+  - id: "deploy to cloud run"
+    name: "gcr.io/cloud-builders/gcloud"
     args: [
       "beta",
       "run",
@@ -26,7 +84,7 @@ steps:
       "--platform", "managed",
       "--allow-unauthenticated",
       "--add-cloudsql-instances", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
-      "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production",
+      "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production,ENV_PATH=/tmp/secrets/.env",
       "--min-instances", "1",
       "--max-instances", "8",
       "--memory", "512M",

config/settings/production.py

@@ -1,7 +1,13 @@
+import io
 import json
 import logging
 
+import google.auth
+import google.auth.exceptions
 import sentry_sdk
+from django.conf import ImproperlyConfigured
+from dotenv import load_dotenv
+from google.cloud import secretmanager
 from google.oauth2 import service_account
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
@@ -14,7 +20,30 @@
 ###############################################################################
 
 ENV_PATH = env.str("ENV_PATH", default="/tmp/secrets/.env")
-env.read_env(path=ENV_PATH, override=True)
+# First, we try and load the environment variables from an .env file if a path
+# to the file is provided.
+if ENV_PATH:
+    env.read_env(path=ENV_PATH, override=True)
+# Else, load the variables from Google Secrets Manager
+else:
+    SETTINGS_NAME = env.str("SETTINGS_NAME")
+    try:
+        GCP_PROJECT_ID = env.str(
+            "GOOGLE_CLOUD_PROJECT", default=google.auth.default()
+        )
+    except google.auth.exceptions.DefaultCredentialsError:
+        raise ImproperlyConfigured(
+            "No local .env or GOOGLE_CLOUD_PROJECT detected. No secrets found."
+        )
+
+    secret_manager_client = secretmanager.SecretManagerServiceClient()
+    secrets_name = "projects/{}/secrets/{}/versions/latest".format(
+        GCP_PROJECT_ID, SETTINGS_NAME
+    )
+    payload = secret_manager_client.access_secret_version(
+        name=secrets_name
+    ).payload.data.decode("UTF-8")
+    load_dotenv(stream=io.StringIO(payload), override=True)
 
 
 ###############################################################################
@@ -119,7 +148,7 @@
 
 INSTALLED_APPS += ["storages"]  # noqa: F405
 GS_BUCKET_NAME = env.str("DJANGO_GCP_STORAGE_BUCKET_NAME")
-GS_DEFAULT_ACL = "project-private"
+GS_DEFAULT_ACL = "projectPrivate"
 
 
 ###############################################################################
@@ -128,7 +157,7 @@
 
 DEFAULT_FILE_STORAGE = "utils.storages.MediaRootGoogleCloudStorage"
 MEDIA_URL = "https://storage.googleapis.com/%s/media/" % GS_BUCKET_NAME
-STATICFILES_STORAGE = "whitenoise.storage.CompressedStaticFilesStorage"
+STATICFILES_STORAGE = "utils.storages.StaticRootGoogleCloudStorage"
 
 
 ###############################################################################

requirements/production.txt

@@ -19,5 +19,6 @@ grpcio~=1.53.0
 # Others
 # -----------------------------------------------------------------------------
 gunicorn~=20.1.0
-sentry-sdk~=1.18.0
+python-dotenv~=1.0.0
+sentry-sdk~=1.19.0
 uvicorn~=0.21.1

utils/storages.py

@@ -2,10 +2,12 @@
 
 
 class StaticRootGoogleCloudStorage(GoogleCloudStorage):
+    default_acl = "publicRead"
+    file_overwrite = True
     location = "static"
-    default_acl = "project-private"
 
 
 class MediaRootGoogleCloudStorage(GoogleCloudStorage):
-    location = "media"
+    default_acl = "projectPrivate"
     file_overwrite = False
+    location = "media"