diff --git a/.github/workflows/deploy_dev.yml b/.github/workflows/deploy_dev.yml index 1948cd22dd..61baefb295 100644 --- a/.github/workflows/deploy_dev.yml +++ b/.github/workflows/deploy_dev.yml @@ -1,7 +1,10 @@ -name: Deploy to dev Heroku +name: Deploy to dev on: workflow_dispatch: + push: + branches: + - main jobs: deploy: @@ -9,6 +12,7 @@ jobs: env: HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} HEROKU_APP: dev-metaculus-web + NEXT_PUBLIC_APP_URL: "https://dev.metaculus.com" steps: - name: Checkout code @@ -16,5 +20,4 @@ jobs: - name: Build and deploy to Heroku run: | heroku container:login # uses the HEROKU_API_KEY - docker buildx build --cache-to=type=gha --cache-from type=gha . # used only for caching ./scripts/deploy_to_heroku.sh diff --git a/.github/workflows/deploy_prod.sh b/.github/workflows/deploy_prod.sh new file mode 100644 index 0000000000..ca0a5e4b19 --- /dev/null +++ b/.github/workflows/deploy_prod.sh @@ -0,0 +1,20 @@ +name: Deploy to PRODUCTION + +on: + workflow_dispatch: + +jobs: + deploy: + runs-on: ubuntu-latest + env: + HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }} + HEROKU_APP: dev-metaculus-web + NEXT_PUBLIC_APP_URL: "https://beta.metaculus.com" + + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Build and deploy to Heroku + run: | + heroku container:login # uses the HEROKU_API_KEY + ./scripts/deploy_to_heroku.sh diff --git a/Dockerfile b/Dockerfile index 12ca0faf23..f0f9211573 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,8 @@ -FROM alpine:latest +FROM alpine:latest AS base -RUN apk add --no-cache --update python3 py3-pip bash curl git - - -# Needed for installing/compiling python via pyenv (so we have the right python version) -RUN apk add --no-cache --update build-base - -RUN apk add --no-cache --update openssl-dev \ +RUN apk add --no-cache --update python3 py3-pip bash curl git \ + build-base \ + openssl-dev \ zlib-dev \ bzip2-dev \ readline-dev \ @@ -39,18 +35,15 @@ ADD front_end/.nvmrc /tmp/.nvmrc # Install Nodejs # Inspired from: https://github.com/nodejs/docker-node/blob/main/Dockerfile-alpine.template ENV ARCH=x64 - RUN export NODE_VERSION=$(cat /tmp/.nvmrc) && cd /tmp/ && curl -fsSLO --compressed "https://unofficial-builds.nodejs.org/download/release/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" \ && tar -xJf "node-v$NODE_VERSION-linux-$ARCH-musl.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ && ln -s /usr/local/bin/node /usr/local/bin/nodejs - -# This is done to copy only the source code from HEAD into the image -RUN --mount=type=bind,source=.git/,target=/tmp/app/.git/ \ - git clone /tmp/app/.git/ /app/ - +FROM base AS backend_deps WORKDIR /app +ADD poetry.lock poetry.lock +ADD pyproject.toml pyproject.toml # Needed so the env created by poetry is saved after the build phase. Don't know of another way RUN poetry config virtualenvs.create false \ && python -m venv venv \ @@ -58,16 +51,41 @@ RUN poetry config virtualenvs.create false \ && poetry install --without dev +FROM base AS frontend_deps +WORKDIR /app/front_end/ + +ADD front_end/package*.json . ENV NODE_ENV=production -RUN cd front_end && npm ci && npm run build +RUN npm ci + + +FROM base AS final_env +WORKDIR /app + +# This is done to copy only the source code from HEAD into the image to avoid a COPY . and managing a long .dockerignore +RUN --mount=type=bind,source=.git/,target=/tmp/app/.git/ \ +git clone /tmp/app/.git/ /app/ + +# Copy the backkend and frontend deps +COPY --from=backend_deps /app/venv /app/venv +COPY --from=frontend_deps /app/front_end/node_modules /app/front_end/node_modules + +ENV NODE_ENV=production +RUN --mount=type=secret,id=frontend_env,target=/app/front_end/.env cd front_end && npm run build RUN source venv/bin/activate && ./manage.py collectstatic --noinput ENV PORT=3000 EXPOSE 3000 -ARG ENTRY_SCRIPT_ARG="scripts/prod/startapp.sh" -ENV ENTRY_SCRIPT=${ENTRY_SCRIPT_ARG} +FROM final_env AS release +CMD ["sh", "-c", "scripts/prod/release.sh"] + +FROM final_env AS web +CMD ["sh", "-c", "scripts/prod/startapp.sh"] -CMD ["sh", "-c", "${ENTRY_SCRIPT}"] +FROM final_env AS django_cron +CMD ["sh", "-c", "scripts/prod/django_cron.sh"] +FROM final_env AS dramatiq_worker +CMD ["sh", "-c", "scripts/prod/run_dramatiq.sh"] diff --git a/scripts/deploy_to_heroku.sh b/scripts/deploy_to_heroku.sh index 808c351c7c..feee8978ed 100755 --- a/scripts/deploy_to_heroku.sh +++ b/scripts/deploy_to_heroku.sh @@ -1,16 +1,31 @@ #! /bin/bash -set -x +required_vars=("NEXT_PUBLIC_TURNSTILE_SITE_KEY" "HEROKU_APP" "NEXT_PUBLIC_APP_URL") -[ -z "$HEROKU_APP" ] && { echo "Error: HEROKU_APP env varible is not set."; exit 1; } +for var in "${required_vars[@]}"; do + if [ -z "${!var}" ]; then + echo "Error: $var environment variable is not set." + exit 1 + fi +done -# Push container -heroku container:push release --arg ENTRY_SCRIPT_ARG="scripts/prod/release.sh" -a $HEROKU_APP -heroku container:push web --arg ENTRY_SCRIPT_ARG="scripts/prod/startapp.sh" -a $HEROKU_APP -heroku container:push dramatiq_worker --arg ENTRY_SCRIPT_ARG="scripts/prod/run_dramatiq.sh" -a $HEROKU_APP -heroku container:push django_cron --arg ENTRY_SCRIPT_ARG="scripts/prod/django_cron.sh" -a $HEROKU_APP +# These are needed for nextjs build phase, as it replaces the value of these environmental variables +# at build time :/ +FRONTEND_ENV_FILE=$(mktemp) +echo NEXT_PUBLIC_TURNSTILE_SITE_KEY=$NEXT_PUBLIC_TURNSTILE_SITE_KEY >$FRONTEND_ENV_FILE +echo NEXT_PUBLIC_APP_URL=$NEXT_PUBLIC_APP_URL >>$FRONTEND_ENV_FILE +docker buildx build \ + --secret id=frontend_env,src=$FRONTEND_ENV_FILE \ + --platform linux/amd64 -t registry.heroku.com/$HEROKU_APP/web --target web . -# Release them -heroku container:release release -a $HEROKU_APP -heroku container:release web -a $HEROKU_APP -heroku container:release dramatiq_worker -a $HEROKU_APP -heroku container:release django_cron -a $HEROKU_APP +# The rest of the target images don't require any special env variables +for target in release dramatiq_worker django_cron; do + docker build --platform linux/amd64 . -t registry.heroku.com/$HEROKU_APP/$target --target $target +done + +# Push all built images to the heroku docker registry +for target in release dramatiq_worker django_cron web; do + docker push registry.heroku.com/$HEROKU_APP/$target +done + +# Release them all +heroku container:release release web dramatiq_worker django_cron -a $HEROKU_APP diff --git a/scripts/prod/release.sh b/scripts/prod/release.sh index d34ae0e5b8..0af680071f 100755 --- a/scripts/prod/release.sh +++ b/scripts/prod/release.sh @@ -3,5 +3,4 @@ cd /app/ source venv/bin/activate -# PORT is passed by Heroku, and should be the one where nextjs lists on -python manage.py migrate \ No newline at end of file +python manage.py migrate diff --git a/scripts/prod/startapp.sh b/scripts/prod/startapp.sh index 80956bbc09..6d3772ad57 100755 --- a/scripts/prod/startapp.sh +++ b/scripts/prod/startapp.sh @@ -1,11 +1,21 @@ -#! /bin/bash +#!/bin/bash +set -e +set -o pipefail -cd /app/ -source venv/bin/activate +cleanup() { + echo "Stopping the processes" + for process in "gunicorn"; do + PID=$(ps -eo pid,comm,args | awk -v process=$process '$0 ~ process && $0 !~ /awk/ {print $1}') + [ -n "$PID" ] && kill -s SIGTERM $PID + done +} -# PORT is passed by Heroku, and should be the one where nextjs lists on -gunicorn metaculus_web.wsgi:application --workers 8 --bind 0.0.0.0:8000 & +trap cleanup EXIT +trap "exit" INT TERM ERR +cd /app/ +source venv/bin/activate -cd front_end -NEXT_PUBLIC_APP_URL=http://localhost:$PORT && npm run start \ No newline at end of file +export NEXT_PUBLIC_APP_URL="http://localhost:$PORT" +(gunicorn metaculus_web.wsgi:application --bind 0.0.0.0:8000 2>&1 | sed 's/^/[Backend]: /') & +(cd front_end && npm run start 2>&1 | sed 's/^/[Frontend]: /')