chore(conf): migrate production static storage to GCS buckets (#62)

Migrate the production static storage from the local filesystem to a Google Cloud Storage bucket. Other changes included are: - Change the deployment process to also perform database migrations and collect static files. The prior implementation would perform this during container startup which would in turn lengthen the container startup by a few more seconds. - Change environment loading behaviour in production to first load the environment variables from a `.env` file if a path to one was provided. If not, load the environment variables from the GCP Secret Manager service. If neither of these is successful, fail with a descriptive error message. Included also are other minor improvements and fixes.
savannahghi · Apr 5, 2023 · 7f96982 · 7f96982
1 parent d3fcd88
commit 7f96982
Show file tree

Hide file tree

Showing 7 changed files with 149 additions and 68 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -6,7 +6,7 @@ ARG APP_HOME=/app
 WORKDIR ${APP_HOME}
 
 COPY . ${APP_HOME}
-RUN npm install && npm cache clean --force
+RUN npm ci --no-audit && npm cache clean --force
 RUN npm run build
 
 # Define an alias for the specfic python version used in this file.
@@ -62,12 +62,15 @@ COPY --from=python-build-stage /usr/src/app/wheels  /wheels/
 RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
   && rm -rf /wheels/
 
-
-COPY ./entrypoint /entrypoint
-RUN sed -i 's/\r$//g' /entrypoint
-RUN chmod +x /entrypoint
-
 # Copy application code to WORKDIR
 COPY --from=client-builder ${APP_HOME} ${APP_HOME}
 
-ENTRYPOINT ["/entrypoint"]
+
+# Run the web service on container startup.
+# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud
+# Run to handle instance scaling.
+CMD gunicorn config.asgi \
+    --bind 0.0.0.0:$PORT \
+    --timeout 0 \
+    --chdir=/app \
+    -k uvicorn.workers.UvicornWorker
diff --git a/cloudbuild.yaml b/cloudbuild.yaml
@@ -1,6 +1,7 @@
 steps:
-  # build the container image
-  - name: "gcr.io/cloud-builders/docker"
+  # Build the container image
+  - id: "build image"
+    name: "gcr.io/cloud-builders/docker"
     args:
       [
         "build",
@@ -10,12 +11,69 @@ steps:
       ]
 
   # Push the container image to Container Registry
-  - name: "gcr.io/cloud-builders/docker"
+  - id: "push image"
+    name: "gcr.io/cloud-builders/docker"
     args:
-      ["push", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA"]
+      [
+        "push",
+        "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA"
+      ]
+
+  # Apply the latest migrations
+  - id: "apply migrations"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "migrate",
+      ]
+
+  # Create cache table
+  - id: "create cache table"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "createcachetable",
+      ]
+
+  # Collect static files
+  - id: "collect static files"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "collectstatic", "--noinput"
+      ]
+
+  # Collect static files
+  - id: "compress static assets"
+    name: "gcr.io/google-appengine/exec-wrapper"
+    args:
+      [
+        "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA",
+        "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
+        "-e", "DJANGO_SETTINGS_MODULE=config.settings.production",
+        "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID",
+        "-e", "SETTINGS_NAME=${_SETTINGS_NAME}",
+        "--", "python", "/app/manage.py", "compress"
+      ]
 
   # Deploy an image from Container Registry to Cloud Run
-  - name: "gcr.io/cloud-builders/gcloud"
+  - id: "deploy to cloud run"
+    name: "gcr.io/cloud-builders/gcloud"
     args: [
       "beta",
       "run",
@@ -26,7 +84,7 @@ steps:
       "--platform", "managed",
       "--allow-unauthenticated",
       "--add-cloudsql-instances", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}",
-      "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production",
+      "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production,ENV_PATH=/tmp/secrets/.env",
       "--min-instances", "1",
       "--max-instances", "8",
       "--memory", "512M",

diff --git a/config/settings/production.py b/config/settings/production.py
@@ -1,7 +1,13 @@
+import io
 import json
 import logging
 
+import google.auth
+import google.auth.exceptions
 import sentry_sdk
+from django.conf import ImproperlyConfigured
+from dotenv import load_dotenv
+from google.cloud import secretmanager
 from google.oauth2 import service_account
 from sentry_sdk.integrations.django import DjangoIntegration
 from sentry_sdk.integrations.logging import LoggingIntegration
@@ -13,8 +19,31 @@
 # READ ENVIRONMENT
 ###############################################################################
 
-ENV_PATH = env.str("ENV_PATH", default="/tmp/secrets/.env")
-env.read_env(path=ENV_PATH, override=True)
+ENV_PATH = env.str("ENV_PATH", default=None)
+# First, try and load the environment variables from an .env file if a path to
+# the file is provided.
+if ENV_PATH:
+    env.read_env(path=ENV_PATH, override=True)
+# Else, load the variables from Google Secrets Manager
+else:
+    SETTINGS_NAME = env.str("SETTINGS_NAME")
+    try:
+        GCP_PROJECT_ID = env.str(
+            "GOOGLE_CLOUD_PROJECT", default=google.auth.default()
+        )
+    except google.auth.exceptions.DefaultCredentialsError:
+        raise ImproperlyConfigured(
+            "No local .env or GOOGLE_CLOUD_PROJECT detected. No secrets found."
+        )
+
+    secret_manager_client = secretmanager.SecretManagerServiceClient()
+    secrets_name = "projects/{}/secrets/{}/versions/latest".format(
+        GCP_PROJECT_ID, SETTINGS_NAME
+    )
+    payload = secret_manager_client.access_secret_version(
+        name=secrets_name
+    ).payload.data.decode("UTF-8")
+    load_dotenv(stream=io.StringIO(payload), override=True)
 
 
 ###############################################################################
@@ -119,7 +148,7 @@
 
 INSTALLED_APPS += ["storages"]  # noqa: F405
 GS_BUCKET_NAME = env.str("DJANGO_GCP_STORAGE_BUCKET_NAME")
-GS_DEFAULT_ACL = "project-private"
+GS_DEFAULT_ACL = "projectPrivate"
 
 
 ###############################################################################
@@ -128,20 +157,15 @@
 
 DEFAULT_FILE_STORAGE = "utils.storages.MediaRootGoogleCloudStorage"
 MEDIA_URL = "https://storage.googleapis.com/%s/media/" % GS_BUCKET_NAME
-STATICFILES_STORAGE = "whitenoise.storage.CompressedStaticFilesStorage"
+STATIC_URL = "https://storage.googleapis.com/%s/static/" % GS_BUCKET_NAME
+STATICFILES_STORAGE = "utils.storages.StaticRootGoogleCloudStorage"
 
 
 ###############################################################################
 # DJANGO COMPRESSOR
 ###############################################################################
 
 COMPRESS_ENABLED = True
-# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_URL
-COMPRESS_URL = STATIC_URL  # noqa F405
-# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE
-COMPRESS_OFFLINE = (
-    True  # Offline compression is required when using Whitenoise
-)
 # https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_FILTERS
 COMPRESS_FILTERS = {
     "css": [
@@ -150,6 +174,14 @@
     ],
     "js": ["compressor.filters.jsmin.JSMinFilter"],
 }
+# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE
+COMPRESS_OFFLINE = True
+# https://django-compressor.readthedocs.io/en/stable/settings.html#django.conf.settings.COMPRESS_OFFLINE_MANIFEST_STORAGE
+COMPRESS_OFFLINE_MANIFEST_STORAGE = STATICFILES_STORAGE
+# https://django-compressor.readthedocs.io/en/stable/settings.html#django.conf.settings.COMPRESS_STORAGE
+COMPRESS_STORAGE = STATICFILES_STORAGE
+# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_URL
+COMPRESS_URL = STATIC_URL
 
 
 ###############################################################################

diff --git a/config/urls.py b/config/urls.py
@@ -41,30 +41,35 @@
             url=settings.STATIC_URL + "favicon.ico", permanent=True
         ),
     ),
-] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
+]
 
 if settings.DEBUG:
     # Static file serving when using Gunicorn + Uvicorn for local web socket
     # development.
     urlpatterns += staticfiles_urlpatterns()
-    if settings.DEBUG:
-        # This allows the error pages to be debugged during development, just
-        # visit these url in browser to see how these error pages look like.
-        urlpatterns += [
-            path(
-                "400/",
-                default_views.bad_request,
-                kwargs={"exception": Exception("Bad Request!")},
-            ),
-            path(
-                "403/",
-                default_views.permission_denied,
-                kwargs={"exception": Exception("Permission Denied")},
-            ),
-            path(
-                "404/",
-                default_views.page_not_found,
-                kwargs={"exception": Exception("Page not Found")},
-            ),
-            path("500/", default_views.server_error),
-        ]
+
+    # Server media files during local development.
+    urlpatterns += static(
+        settings.MEDIA_URL, document_root=settings.MEDIA_ROOT
+    )
+
+    # This allows the error pages to be debugged during development, just
+    # visit these url in browser to see how these error pages look like.
+    urlpatterns += [
+        path(
+            "400/",
+            default_views.bad_request,
+            kwargs={"exception": Exception("Bad Request!")},
+        ),
+        path(
+            "403/",
+            default_views.permission_denied,
+            kwargs={"exception": Exception("Permission Denied")},
+        ),
+        path(
+            "404/",
+            default_views.page_not_found,
+            kwargs={"exception": Exception("Page not Found")},
+        ),
+        path("500/", default_views.server_error),
+    ]
diff --git a/entrypoint b/entrypoint
diff --git a/requirements/production.txt b/requirements/production.txt
@@ -19,5 +19,6 @@ grpcio~=1.53.0
 # Others
 # -----------------------------------------------------------------------------
 gunicorn~=20.1.0
-sentry-sdk~=1.18.0
+python-dotenv~=1.0.0
+sentry-sdk~=1.19.0
 uvicorn~=0.21.1
diff --git a/utils/storages.py b/utils/storages.py
@@ -2,10 +2,12 @@
 
 
 class StaticRootGoogleCloudStorage(GoogleCloudStorage):
+    default_acl = "publicRead"
+    file_overwrite = True
     location = "static"
-    default_acl = "project-private"
 
 
 class MediaRootGoogleCloudStorage(GoogleCloudStorage):
-    location = "media"
+    default_acl = "projectPrivate"
     file_overwrite = False
+    location = "media"