From 7f9698259740d2473ec52f122d54a10f736d0ed1 Mon Sep 17 00:00:00 2001 From: Kennedy Kori Date: Wed, 5 Apr 2023 06:13:58 +0300 Subject: [PATCH] chore(conf): migrate production static storage to GCS buckets (#62) Migrate the production static storage from the local filesystem to a Google Cloud Storage bucket. Other changes included are: - Change the deployment process to also perform database migrations and collect static files. The prior implementation would perform this during container startup which would in turn lengthen the container startup by a few more seconds. - Change environment loading behaviour in production to first load the environment variables from a `.env` file if a path to one was provided. If not, load the environment variables from the GCP Secret Manager service. If neither of these is successful, fail with a descriptive error message. Included also are other minor improvements and fixes. --- Dockerfile | 17 +++++---- cloudbuild.yaml | 70 ++++++++++++++++++++++++++++++++--- config/settings/production.py | 52 +++++++++++++++++++++----- config/urls.py | 49 +++++++++++++----------- entrypoint | 20 ---------- requirements/production.txt | 3 +- utils/storages.py | 6 ++- 7 files changed, 149 insertions(+), 68 deletions(-) delete mode 100644 entrypoint diff --git a/Dockerfile b/Dockerfile index 0f3dec4..759a1c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG APP_HOME=/app WORKDIR ${APP_HOME} COPY . ${APP_HOME} -RUN npm install && npm cache clean --force +RUN npm ci --no-audit && npm cache clean --force RUN npm run build # Define an alias for the specfic python version used in this file. @@ -62,12 +62,15 @@ COPY --from=python-build-stage /usr/src/app/wheels /wheels/ RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \ && rm -rf /wheels/ - -COPY ./entrypoint /entrypoint -RUN sed -i 's/\r$//g' /entrypoint -RUN chmod +x /entrypoint - # Copy application code to WORKDIR COPY --from=client-builder ${APP_HOME} ${APP_HOME} -ENTRYPOINT ["/entrypoint"] + +# Run the web service on container startup. +# Timeout is set to 0 to disable the timeouts of the workers to allow Cloud +# Run to handle instance scaling. +CMD gunicorn config.asgi \ + --bind 0.0.0.0:$PORT \ + --timeout 0 \ + --chdir=/app \ + -k uvicorn.workers.UvicornWorker diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 8c0d9c2..f427397 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,6 +1,7 @@ steps: - # build the container image - - name: "gcr.io/cloud-builders/docker" + # Build the container image + - id: "build image" + name: "gcr.io/cloud-builders/docker" args: [ "build", @@ -10,12 +11,69 @@ steps: ] # Push the container image to Container Registry - - name: "gcr.io/cloud-builders/docker" + - id: "push image" + name: "gcr.io/cloud-builders/docker" args: - ["push", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA"] + [ + "push", + "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA" + ] + + # Apply the latest migrations + - id: "apply migrations" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA", + "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}", + "-e", "DJANGO_SETTINGS_MODULE=config.settings.production", + "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID", + "-e", "SETTINGS_NAME=${_SETTINGS_NAME}", + "--", "python", "/app/manage.py", "migrate", + ] + + # Create cache table + - id: "create cache table" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA", + "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}", + "-e", "DJANGO_SETTINGS_MODULE=config.settings.production", + "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID", + "-e", "SETTINGS_NAME=${_SETTINGS_NAME}", + "--", "python", "/app/manage.py", "createcachetable", + ] + + # Collect static files + - id: "collect static files" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA", + "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}", + "-e", "DJANGO_SETTINGS_MODULE=config.settings.production", + "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID", + "-e", "SETTINGS_NAME=${_SETTINGS_NAME}", + "--", "python", "/app/manage.py", "collectstatic", "--noinput" + ] + + # Collect static files + - id: "compress static assets" + name: "gcr.io/google-appengine/exec-wrapper" + args: + [ + "-i", "europe-west1-docker.pkg.dev/$PROJECT_ID/fyj/idr-server-${_DEPLOYMENT_TYPE}:$COMMIT_SHA", + "-s", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}", + "-e", "DJANGO_SETTINGS_MODULE=config.settings.production", + "-e", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID", + "-e", "SETTINGS_NAME=${_SETTINGS_NAME}", + "--", "python", "/app/manage.py", "compress" + ] # Deploy an image from Container Registry to Cloud Run - - name: "gcr.io/cloud-builders/gcloud" + - id: "deploy to cloud run" + name: "gcr.io/cloud-builders/gcloud" args: [ "beta", "run", @@ -26,7 +84,7 @@ steps: "--platform", "managed", "--allow-unauthenticated", "--add-cloudsql-instances", "${_CLOUDSQL_INSTANCE_CONNECTION_NAME}", - "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production", + "--set-env-vars", "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,SETTINGS_NAME=${_SETTINGS_NAME},DJANGO_SETTINGS_MODULE=config.settings.production,ENV_PATH=/tmp/secrets/.env", "--min-instances", "1", "--max-instances", "8", "--memory", "512M", diff --git a/config/settings/production.py b/config/settings/production.py index 38aceaf..54f7143 100644 --- a/config/settings/production.py +++ b/config/settings/production.py @@ -1,7 +1,13 @@ +import io import json import logging +import google.auth +import google.auth.exceptions import sentry_sdk +from django.conf import ImproperlyConfigured +from dotenv import load_dotenv +from google.cloud import secretmanager from google.oauth2 import service_account from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.logging import LoggingIntegration @@ -13,8 +19,31 @@ # READ ENVIRONMENT ############################################################################### -ENV_PATH = env.str("ENV_PATH", default="/tmp/secrets/.env") -env.read_env(path=ENV_PATH, override=True) +ENV_PATH = env.str("ENV_PATH", default=None) +# First, try and load the environment variables from an .env file if a path to +# the file is provided. +if ENV_PATH: + env.read_env(path=ENV_PATH, override=True) +# Else, load the variables from Google Secrets Manager +else: + SETTINGS_NAME = env.str("SETTINGS_NAME") + try: + GCP_PROJECT_ID = env.str( + "GOOGLE_CLOUD_PROJECT", default=google.auth.default() + ) + except google.auth.exceptions.DefaultCredentialsError: + raise ImproperlyConfigured( + "No local .env or GOOGLE_CLOUD_PROJECT detected. No secrets found." + ) + + secret_manager_client = secretmanager.SecretManagerServiceClient() + secrets_name = "projects/{}/secrets/{}/versions/latest".format( + GCP_PROJECT_ID, SETTINGS_NAME + ) + payload = secret_manager_client.access_secret_version( + name=secrets_name + ).payload.data.decode("UTF-8") + load_dotenv(stream=io.StringIO(payload), override=True) ############################################################################### @@ -119,7 +148,7 @@ INSTALLED_APPS += ["storages"] # noqa: F405 GS_BUCKET_NAME = env.str("DJANGO_GCP_STORAGE_BUCKET_NAME") -GS_DEFAULT_ACL = "project-private" +GS_DEFAULT_ACL = "projectPrivate" ############################################################################### @@ -128,7 +157,8 @@ DEFAULT_FILE_STORAGE = "utils.storages.MediaRootGoogleCloudStorage" MEDIA_URL = "https://storage.googleapis.com/%s/media/" % GS_BUCKET_NAME -STATICFILES_STORAGE = "whitenoise.storage.CompressedStaticFilesStorage" +STATIC_URL = "https://storage.googleapis.com/%s/static/" % GS_BUCKET_NAME +STATICFILES_STORAGE = "utils.storages.StaticRootGoogleCloudStorage" ############################################################################### @@ -136,12 +166,6 @@ ############################################################################### COMPRESS_ENABLED = True -# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_URL -COMPRESS_URL = STATIC_URL # noqa F405 -# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE -COMPRESS_OFFLINE = ( - True # Offline compression is required when using Whitenoise -) # https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_FILTERS COMPRESS_FILTERS = { "css": [ @@ -150,6 +174,14 @@ ], "js": ["compressor.filters.jsmin.JSMinFilter"], } +# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_OFFLINE +COMPRESS_OFFLINE = True +# https://django-compressor.readthedocs.io/en/stable/settings.html#django.conf.settings.COMPRESS_OFFLINE_MANIFEST_STORAGE +COMPRESS_OFFLINE_MANIFEST_STORAGE = STATICFILES_STORAGE +# https://django-compressor.readthedocs.io/en/stable/settings.html#django.conf.settings.COMPRESS_STORAGE +COMPRESS_STORAGE = STATICFILES_STORAGE +# https://django-compressor.readthedocs.io/en/latest/settings/#django.conf.settings.COMPRESS_URL +COMPRESS_URL = STATIC_URL ############################################################################### diff --git a/config/urls.py b/config/urls.py index ca7be6b..21e8144 100644 --- a/config/urls.py +++ b/config/urls.py @@ -41,30 +41,35 @@ url=settings.STATIC_URL + "favicon.ico", permanent=True ), ), -] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) +] if settings.DEBUG: # Static file serving when using Gunicorn + Uvicorn for local web socket # development. urlpatterns += staticfiles_urlpatterns() - if settings.DEBUG: - # This allows the error pages to be debugged during development, just - # visit these url in browser to see how these error pages look like. - urlpatterns += [ - path( - "400/", - default_views.bad_request, - kwargs={"exception": Exception("Bad Request!")}, - ), - path( - "403/", - default_views.permission_denied, - kwargs={"exception": Exception("Permission Denied")}, - ), - path( - "404/", - default_views.page_not_found, - kwargs={"exception": Exception("Page not Found")}, - ), - path("500/", default_views.server_error), - ] + + # Server media files during local development. + urlpatterns += static( + settings.MEDIA_URL, document_root=settings.MEDIA_ROOT + ) + + # This allows the error pages to be debugged during development, just + # visit these url in browser to see how these error pages look like. + urlpatterns += [ + path( + "400/", + default_views.bad_request, + kwargs={"exception": Exception("Bad Request!")}, + ), + path( + "403/", + default_views.permission_denied, + kwargs={"exception": Exception("Permission Denied")}, + ), + path( + "404/", + default_views.page_not_found, + kwargs={"exception": Exception("Page not Found")}, + ), + path("500/", default_views.server_error), + ] diff --git a/entrypoint b/entrypoint deleted file mode 100644 index dbf3d28..0000000 --- a/entrypoint +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -set -o errexit -set -o pipefail -set -o nounset - -python /app/manage.py migrate ->&2 echo 'Ran database migrations...' - -python /app/manage.py createcachetable ->&2 echo 'Created the cache table...' - -python /app/manage.py collectstatic --noinput ->&2 echo 'Collected static files...' - -python /app/manage.py compress ->&2 echo 'Compressed static assets...' - ->&2 echo 'About to run Gunicorn...' -/usr/local/bin/gunicorn config.asgi --bind 0.0.0.0:$PORT --chdir=/app -k uvicorn.workers.UvicornWorker diff --git a/requirements/production.txt b/requirements/production.txt index 69663d8..f5f57fb 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -19,5 +19,6 @@ grpcio~=1.53.0 # Others # ----------------------------------------------------------------------------- gunicorn~=20.1.0 -sentry-sdk~=1.18.0 +python-dotenv~=1.0.0 +sentry-sdk~=1.19.0 uvicorn~=0.21.1 diff --git a/utils/storages.py b/utils/storages.py index 78952dc..d3f5f70 100644 --- a/utils/storages.py +++ b/utils/storages.py @@ -2,10 +2,12 @@ class StaticRootGoogleCloudStorage(GoogleCloudStorage): + default_acl = "publicRead" + file_overwrite = True location = "static" - default_acl = "project-private" class MediaRootGoogleCloudStorage(GoogleCloudStorage): - location = "media" + default_acl = "projectPrivate" file_overwrite = False + location = "media"