Gh deployment action

.github/workflows/deploy_dev.yml

@@ -1,20 +1,23 @@
-name: Deploy to dev Heroku
+name: Deploy to dev
 
 on:
   workflow_dispatch:
+  push:
+    branches:
+      - main
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
     env:
       HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
       HEROKU_APP: dev-metaculus-web
+      NEXT_PUBLIC_APP_URL: "https://dev.metaculus.com"
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Build and deploy to Heroku
         run: |
           heroku container:login # uses the HEROKU_API_KEY
-          docker buildx build --cache-to=type=gha --cache-from type=gha . # used only for caching
           ./scripts/deploy_to_heroku.sh

.github/workflows/deploy_prod.sh

@@ -0,0 +1,20 @@
+name: Deploy to PRODUCTION
+
+on:
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    env:
+      HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+      HEROKU_APP: dev-metaculus-web
+      NEXT_PUBLIC_APP_URL: "https://beta.metaculus.com"
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Build and deploy to Heroku
+        run: |
+          heroku container:login # uses the HEROKU_API_KEY
+          ./scripts/deploy_to_heroku.sh

Dockerfile

@@ -1,12 +1,8 @@
-FROM alpine:latest
+FROM alpine:latest AS base
 
-RUN apk add --no-cache --update python3 py3-pip bash curl git
-
-
-# Needed for installing/compiling python via pyenv (so we have the right python version)
-RUN apk add --no-cache --update build-base
-
-RUN apk add --no-cache --update openssl-dev \
+RUN apk add --no-cache --update python3 py3-pip bash curl git \
+    build-base \
+    openssl-dev \
     zlib-dev \
     bzip2-dev \
     readline-dev \
@@ -39,35 +35,57 @@ ADD front_end/.nvmrc /tmp/.nvmrc
 # Install Nodejs
 # Inspired from: https://github.com/nodejs/docker-node/blob/main/Dockerfile-alpine.template
 ENV ARCH=x64
-
 RUN export NODE_VERSION=$(cat /tmp/.nvmrc) && cd /tmp/ && curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" \
   && tar -xJf "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
   && ln -s /usr/local/bin/node /usr/local/bin/nodejs
 
-
-# This is done to copy only the source code from HEAD into the image
-RUN --mount=type=bind,source=.git/,target=/tmp/app/.git/ \
-    git clone /tmp/app/.git/ /app/
-
+FROM base AS backend_deps
 WORKDIR /app
 
+ADD poetry.lock poetry.lock
+ADD pyproject.toml pyproject.toml
 # Needed so the env created by poetry is saved after the build phase. Don't know of another way
 RUN poetry config virtualenvs.create false \
     && python -m venv venv \
     && . venv/bin/activate \
     && poetry install --without dev
 
 
+FROM base AS frontend_deps
+WORKDIR /app/front_end/
+
+ADD front_end/package*.json .
 ENV NODE_ENV=production
-RUN cd front_end  && npm ci && npm run build
+RUN npm ci
+
+
+FROM base AS final_env
+WORKDIR /app
+
+# This is done to copy only the source code from HEAD into the image to avoid a COPY . and managing a long .dockerignore
+RUN --mount=type=bind,source=.git/,target=/tmp/app/.git/ \
+git clone /tmp/app/.git/ /app/
+
+# Copy the backkend and frontend deps
+COPY --from=backend_deps /app/venv /app/venv
+COPY --from=frontend_deps /app/front_end/node_modules /app/front_end/node_modules
+
+ENV NODE_ENV=production
+RUN --mount=type=secret,id=frontend_env,target=/app/front_end/.env cd front_end && npm run build
 
 RUN source venv/bin/activate && ./manage.py collectstatic --noinput
 
 ENV PORT=3000
 EXPOSE 3000
 
-ARG ENTRY_SCRIPT_ARG="scripts/prod/startapp.sh"
-ENV ENTRY_SCRIPT=${ENTRY_SCRIPT_ARG}
+FROM final_env AS release
+CMD ["sh", "-c", "scripts/prod/release.sh"]
+
+FROM final_env AS web
+CMD ["sh", "-c", "scripts/prod/startapp.sh"]
 
-CMD ["sh", "-c", "${ENTRY_SCRIPT}"]
+FROM final_env AS django_cron
+CMD ["sh", "-c", "scripts/prod/django_cron.sh"]
 
+FROM final_env AS dramatiq_worker
+CMD ["sh", "-c", "scripts/prod/run_dramatiq.sh"]

scripts/deploy_to_heroku.sh

@@ -1,16 +1,31 @@
 #! /bin/bash
-set -x
+required_vars=("NEXT_PUBLIC_TURNSTILE_SITE_KEY" "HEROKU_APP" "NEXT_PUBLIC_APP_URL")
 
-[ -z "$HEROKU_APP" ] && { echo "Error: HEROKU_APP env varible is not set."; exit 1; }
+for var in "${required_vars[@]}"; do
+    if [ -z "${!var}" ]; then
+        echo "Error: $var environment variable is not set."
+        exit 1
+    fi
+done
 
-# Push container
-heroku container:push release --arg ENTRY_SCRIPT_ARG="scripts/prod/release.sh" -a $HEROKU_APP
-heroku container:push web --arg ENTRY_SCRIPT_ARG="scripts/prod/startapp.sh" -a $HEROKU_APP
-heroku container:push dramatiq_worker --arg ENTRY_SCRIPT_ARG="scripts/prod/run_dramatiq.sh" -a $HEROKU_APP
-heroku container:push django_cron --arg ENTRY_SCRIPT_ARG="scripts/prod/django_cron.sh" -a $HEROKU_APP
+# These are needed for nextjs build phase, as it replaces the value of these environmental variables
+# at build time :/
+FRONTEND_ENV_FILE=$(mktemp)
+echo NEXT_PUBLIC_TURNSTILE_SITE_KEY=$NEXT_PUBLIC_TURNSTILE_SITE_KEY >$FRONTEND_ENV_FILE
+echo NEXT_PUBLIC_APP_URL=$NEXT_PUBLIC_APP_URL >>$FRONTEND_ENV_FILE
+docker buildx build \
+    --secret id=frontend_env,src=$FRONTEND_ENV_FILE \
+    --platform linux/amd64 -t registry.heroku.com/$HEROKU_APP/web --target web .
 
-# Release them
-heroku container:release release -a $HEROKU_APP
-heroku container:release web -a $HEROKU_APP
-heroku container:release dramatiq_worker -a $HEROKU_APP
-heroku container:release django_cron -a $HEROKU_APP
+# The rest of the target images don't require any special env variables
+for target in release dramatiq_worker django_cron; do
+    docker build --platform linux/amd64 . -t registry.heroku.com/$HEROKU_APP/$target --target $target
+done
+
+# Push all built images to the heroku docker registry
+for target in release dramatiq_worker django_cron web; do
+    docker push registry.heroku.com/$HEROKU_APP/$target
+done
+
+# Release them all
+heroku container:release release web dramatiq_worker django_cron -a $HEROKU_APP

scripts/prod/release.sh

@@ -3,5 +3,4 @@
 cd /app/
 source venv/bin/activate
 
-# PORT is passed by Heroku, and should be the one where nextjs lists on
-python manage.py migrate
+python manage.py migrate

scripts/prod/startapp.sh

@@ -1,11 +1,21 @@
-#! /bin/bash
+#!/bin/bash
+set -e
+set -o pipefail
 
-cd /app/
-source venv/bin/activate
+cleanup() {
+    echo "Stopping the processes"
+    for process in "gunicorn"; do
+        PID=$(ps -eo pid,comm,args | awk -v process=$process '$0 ~ process && $0 !~ /awk/ {print $1}')
+        [ -n "$PID" ] && kill -s SIGTERM $PID
+    done
+}
 
-# PORT is passed by Heroku, and should be the one where nextjs lists on
-gunicorn metaculus_web.wsgi:application --workers 8 --bind 0.0.0.0:8000 &
+trap cleanup EXIT
+trap "exit" INT TERM ERR
 
+cd /app/
+source venv/bin/activate
 
-cd front_end
-NEXT_PUBLIC_APP_URL=http://localhost:$PORT && npm run start
+export NEXT_PUBLIC_APP_URL="http://localhost:$PORT"
+(gunicorn metaculus_web.wsgi:application --bind 0.0.0.0:8000 2>&1 | sed 's/^/[Backend]: /') &
+(cd front_end && npm run start 2>&1 | sed 's/^/[Frontend]: /')