diff --git a/.dockerignore b/.dockerignore index 582b84b77..4af633a25 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,3 +5,14 @@ ./*/env.example ./*/.vscode ./*/node_modules +.gitignore +./CONTRIBUTING.md +./docker-compose.yml +./*/docker-compose.yml +./Dockerfile +./Dockerfile.dev +./LICENSE +./README.md +./*/README.* +./dev.sh +./setup_utils/install.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6800e844..72bd38330 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -33,7 +33,17 @@ jobs: uses: aws-actions/amazon-ecr-login@v2 with: registry-type: public - + + - name: Set up Docker + uses: crazy-max/ghaction-setup-docker@v3 + with: + daemon-config: | + { + "debug": true, + "features": { + "containerd-snapshotter": true + } + } - name: Build Docker image run: docker build --tag middleware:latest --platform linux/amd64,linux/arm64 . diff --git a/Dockerfile b/Dockerfile index 24ec40954..37dfc0a41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,99 +1,95 @@ +ARG ENVIRONMENT=prod +ARG POSTGRES_DB_ENABLED=true +ARG DB_INIT_ENABLED=true +ARG REDIS_ENABLED=true +ARG BACKEND_ENABLED=true +ARG FRONTEND_ENABLED=true +ARG CRON_ENABLED=true + # Build the backend -FROM python:3.9-slim as backend-build +FROM python:3.9.19-alpine3.19 as backend-build # Prevents Python from writing pyc files. ENV PYTHONDONTWRITEBYTECODE=1 -ARG ENVIRONMENT=dev +WORKDIR /app/ +COPY ./apiserver /app/apiserver +RUN apk update && \ + apk add --no-cache \ + git \ + postgresql-dev \ + gcc \ + musl-dev \ + python3-dev \ + && cd ./apiserver \ + && python3 -m venv /opt/venv \ + && /opt/venv/bin/pip install --upgrade pip \ + && /opt/venv/bin/pip install -r requirements.txt -WORKDIR /app/backend -COPY ./apiserver /app/backend/apiserver -RUN apt-get update && apt-get install -y --no-install-recommends \ - git \ - libpq-dev \ - gcc \ - build-essential \ - && cd ./apiserver \ - && python3 -m venv /opt/venv \ - && /opt/venv/bin/pip install --upgrade pip \ - && if [ "$ENVIRONMENT" = "dev" ]; then \ - /opt/venv/bin/pip install -r dev-requirements.txt -r requirements.txt; \ - fi - -# Build the frontend -FROM node:16-alpine as frontend-build -WORKDIR /app/frontend/ -COPY ./web-server /app/frontend/web-server -RUN apk add --no-cache yarn && \ - cd web-server && \ - if [ "$ENVIRONMENT" = "dev" ]; then \ - yarn install && yarn cache clean \ - else \ - yarn install && \ - yarn build && \ - yarn cache clean && \ - tar -xvf ./artifacts/artifact.tar.gz --directory=./artifacts && \ - find ./ -mindepth 1 -maxdepth 1 ! -name 'artifacts' -exec rm -rf {} + \ - mv ./artifacts/* ./* && rm -rf ./artifacts; \ - fi +FROM node:16.20.2-alpine3.18 as node # Final image -FROM python:3.9-slim +FROM postgres:alpine3.19 WORKDIR /app COPY --from=backend-build /opt/venv /opt/venv -COPY --from=backend-build /app/backend /app/backend -COPY --from=frontend-build /app/frontend/web-server/ /app/frontend/web-server +COPY --from=backend-build /usr/local/bin/ /usr/local/bin/ +COPY --from=backend-build /usr/local/lib/ /usr/local/lib/ +COPY --from=backend-build /usr/local/include /usr/local/include -COPY ./database-docker/db/ /app/db/ -COPY ./init_db.sh /app/ -COPY ./generate_config_ini.sh /app/ -COPY ./cronjob.txt /etc/cron.d/cronjob -COPY ./supervisord.conf /etc/supervisord.conf +COPY --from=node /usr/lib /usr/lib +COPY --from=node /usr/local/lib /usr/local/lib +COPY --from=node /usr/local/include /usr/local/include +COPY --from=node /usr/local/bin /usr/local/bin -RUN chmod +x /app/init_db.sh \ - && apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - build-essential \ - libpq-dev \ - cron \ - postgresql \ - postgresql-contrib \ - redis-server \ - supervisor \ - curl \ - && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ - && apt-get install -y nodejs \ - && npm install --global yarn \ - && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \ - && chmod +x /usr/local/bin/dbmate \ - && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/15/main/pg_hba.conf \ - && echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf \ - && mkdir -p /var/log/postgres \ - && touch /var/log/postgres/postgres.log \ - && mkdir -p /var/log/init_db \ - && touch /var/log/init_db/init_db.log \ - && mkdir -p /var/log/redis \ - && touch /var/log/redis/redis.log \ - && mkdir -p /var/log/apiserver \ - && touch /var/log/apiserver/apiserver.log \ - && mkdir -p /var/log/webserver \ - && touch /var/log/webserver/webserver.log \ - && mkdir -p /var/log/cron \ - && touch /var/log/cron/cron.log \ - && chmod 0644 /etc/cron.d/cronjob \ - && crontab /etc/cron.d/cronjob \ - && yarn cache clean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* \ - && /app/generate_config_ini.sh -t /app/backend/apiserver/dora/config +COPY . /app/ -ARG POSTGRES_DB_ENABLED=true -ARG DB_INIT_ENABLED=true -ARG REDIS_ENABLED=true -ARG BACKEND_ENABLED=true -ARG FRONTEND_ENABLED=true -ARG CRON_ENABLED=true +RUN apk add --no-cache \ + gcc \ + build-base \ + libpq \ + postgresql-dev \ + postgresql-client \ + postgresql-contrib \ + redis \ + supervisor \ + curl \ + && mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob \ + && chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh \ + && mv ./setup_utils/supervisord.conf /etc/supervisord.conf \ + && mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ \ + && su - postgres -c "initdb -D /var/lib/postgresql/data" \ + && su - postgres -c "echo \"host all all 0.0.0.0/0 md5\" >> /var/lib/postgresql/data/pg_hba.conf" \ + && su - postgres -c "echo \"listen_addresses='*'\" >> /var/lib/postgresql/data/postgresql.conf" \ + && su - postgres -c "rm -f /var/lib/postgresql/data/postmaster.pid" \ + && npm install --global yarn --force \ + && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \ + && chmod +x /usr/local/bin/dbmate \ + && mkdir -p /var/log/postgres \ + && touch /var/log/postgres/postgres.log \ + && mkdir -p /var/log/init_db \ + && touch /var/log/init_db/init_db.log \ + && mkdir -p /var/log/redis \ + && touch /var/log/redis/redis.log \ + && mkdir -p /var/log/apiserver \ + && touch /var/log/apiserver/apiserver.log \ + && mkdir -p /var/log/webserver \ + && touch /var/log/webserver/webserver.log \ + && mkdir -p /var/log/cron \ + && touch /var/log/cron/cron.log \ + && chmod 0644 /etc/cron.d/cronjob \ + && crontab /etc/cron.d/cronjob \ + && /app/setup_utils/generate_config_ini.sh -t /app/apiserver/dora/config \ + && cd /app/web-server \ + && yarn && yarn build \ + && rm -rf ./artifacts \ + && cd /app/ \ + && tar cfz web-server.tar.gz ./web-server \ + && rm -rf ./web-server && mkdir -p /app/web-server \ + && tar cfz /opt/venv.tar.gz /opt/venv/ \ + && rm -rf /opt/venv && mkdir -p /opt/venv \ + && yarn cache clean \ + && rm -rf /var/cache/apk/* ENV POSTGRES_DB_ENABLED=$POSTGRES_DB_ENABLED ENV DB_INIT_ENABLED=$DB_INIT_ENABLED @@ -101,8 +97,19 @@ ENV REDIS_ENABLED=$REDIS_ENABLED ENV BACKEND_ENABLED=$BACKEND_ENABLED ENV FRONTEND_ENABLED=$FRONTEND_ENABLED ENV CRON_ENABLED=$CRON_ENABLED -ENV PATH="/opt/venv/bin:/usr/lib/postgresql/15/bin:$PATH" +ENV DB_HOST=localhost +ENV DB_NAME=dora-oss +ENV DB_PASS=postgres +ENV DB_PORT=5432 +ENV DB_USER=postgres +ENV REDIS_HOST=localhost +ENV REDIS_PORT=6379 +ENV PORT=3000 +ENV NEXT_PUBLIC_APP_ENVIRONMENT="staging" +ENV INTERNAL_API_BASE_URL=http://localhost:9696 +ENV NEXT_PUBLIC_APP_ENVIRONMENT="prod" +ENV PATH="/opt/venv/bin:$PATH" EXPOSE 5432 6379 9696 3000 -CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] +CMD ["/bin/sh", "-c", "/app/setup_utils/start.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index 09d1a184d..8d276c2aa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,6 @@ services: dora-metrics: - container_name: dora-metrics + container_name: middleware build: context: ./ dockerfile: Dockerfile diff --git a/cronjob.txt b/setup_utils/cronjob.txt similarity index 100% rename from cronjob.txt rename to setup_utils/cronjob.txt diff --git a/generate_config_ini.sh b/setup_utils/generate_config_ini.sh similarity index 98% rename from generate_config_ini.sh rename to setup_utils/generate_config_ini.sh index 0a7185b50..30b0963c9 100755 --- a/generate_config_ini.sh +++ b/setup_utils/generate_config_ini.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh # Parse command line arguments while [[ $# -gt 0 ]]; do diff --git a/init_db.sh b/setup_utils/init_db.sh old mode 100644 new mode 100755 similarity index 98% rename from init_db.sh rename to setup_utils/init_db.sh index 4fd049151..72bac2f9a --- a/init_db.sh +++ b/setup_utils/init_db.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/sh set -e set -u diff --git a/install.sh b/setup_utils/install.sh similarity index 100% rename from install.sh rename to setup_utils/install.sh diff --git a/setup_utils/start.sh b/setup_utils/start.sh new file mode 100644 index 000000000..7b736f0e9 --- /dev/null +++ b/setup_utils/start.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +echo 'MHQ_EXTRACT_BACKEND_DEPENDENCIES' +if [ -f /opt/venv.tar.gz ]; then + mkdir -p /opt/venv + tar xzf /opt/venv.tar.gz -C /opt/venv --strip-components=2 + rm -rf /opt/venv.tar.gz +else + echo "Tar file /opt/venv.tar.gz does not exist. Skipping extraction." +fi + +echo 'MHQ_EXTRACT_FRONTEND' +if [ -f /app/web-server.tar.gz ]; then + mkdir -p /app/frontend/web-server + tar xzf /app/web-server.tar.gz -C /app/frontend/web-server --strip-components=2 + rm -rf /app/web-server.tar.gz +else + echo "Tar file /app/web-server.tar.gz does not exist. Skipping extraction." +fi + +pg_ctl stop -D /var/lib/postgres/data; + +echo 'MHQ_STARTING SUPERVISOR' +/usr/bin/supervisord -c /etc/supervisord.conf diff --git a/supervisord.conf b/setup_utils/supervisord.conf similarity index 78% rename from supervisord.conf rename to setup_utils/supervisord.conf index b0f941c85..ec7325961 100644 --- a/supervisord.conf +++ b/setup_utils/supervisord.conf @@ -3,19 +3,19 @@ nodaemon=true [program:postgres] user=postgres -command=postgres -D /var/lib/postgresql/15/main -c config_file=/etc/postgresql/15/main/postgresql.conf +command=/bin/sh -c "pg_ctl start -D /var/lib/postgresql/data" stdout_logfile=/var/log/postgres/postgres.log stdout_logfile_maxbytes=512KB stderr_logfile=/var/log/postgres/postgres.log stderr_logfile_maxbytes=512KB -autorestart=true +autorestart=false environment=POSTGRES_DB_ENABLED=%(ENV_POSTGRES_DB_ENABLED)s autostart=%(ENV_POSTGRES_DB_ENABLED)s [program:initialize_db] -command=/app/init_db.sh +command=/app/setup_utils/init_db.sh directory=/app -startsecs=0 +startsecs=5 stdout_logfile=/var/log/init_db/init_db.log stdout_logfile_maxbytes=512KB stderr_logfile=/var/log/init_db/init_db.log @@ -35,8 +35,8 @@ environment=REDIS_ENABLED=%(ENV_REDIS_ENABLED)s autostart=%(ENV_REDIS_ENABLED)s [program:backend] -command=/bin/bash -c "/opt/venv/bin/pip install --upgrade pip && /opt/venv/bin/pip install -r requirements.txt && /opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9696 --reload app:app" -directory=/app/backend/apiserver +command=/bin/sh -c "/opt/venv/bin/pip install --upgrade pip && /opt/venv/bin/pip install -r requirements.txt && /opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9696 --reload app:app" +directory=/app/apiserver stdout_logfile=/var/log/apiserver/apiserver.log stdout_logfile_maxbytes=512KB stderr_logfile=/var/log/apiserver/apiserver.log @@ -48,8 +48,8 @@ environment=BACKEND_ENABLED=%(ENV_BACKEND_ENABLED)s autostart=%(ENV_BACKEND_ENABLED)s [program:frontend] -command=/bin/bash -c "yarn && yarn cache clean && yarn http" -directory=/app/frontend/web-server +command=/bin/sh -c "yarn http" +directory=/app/web-server stdout_logfile=/var/log/webserver/webserver.log stdout_logfile_maxbytes=512KB stderr_logfile=/var/log/webserver/webserver.log @@ -61,7 +61,7 @@ environment=FRONTEND_ENABLED=%(ENV_FRONTEND_ENABLED)s autostart=%(ENV_FRONTEND_ENABLED)s [program:cron] -command=/usr/sbin/cron -f -l 1 +command=/usr/sbin/crond -f -l 1 autorestart=true startsecs=10 stderr_logfile=/var/log/cron.log diff --git a/web-server/scripts/build.sh b/web-server/scripts/build.sh index e30c66ade..5e1d3bcd4 100755 --- a/web-server/scripts/build.sh +++ b/web-server/scripts/build.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh set -e DIR=$(dirname $0) @@ -13,4 +13,4 @@ yarn run next build echo "EXITED $?" rm -rf .next/cache -yarn run zip \ No newline at end of file +yarn run zip diff --git a/web-server/scripts/server-init.sh b/web-server/scripts/server-init.sh index 98a4ec03d..e205e5826 100755 --- a/web-server/scripts/server-init.sh +++ b/web-server/scripts/server-init.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh set -e DIR=$(dirname $0) @@ -14,4 +14,4 @@ pm2 delete MHQ_HTTP_SERVER set -e NEXT_MANUAL_SIG_HANDLE=true -pm2 start "yarn http" --name MHQ_HTTP_SERVER \ No newline at end of file +pm2 start "yarn http" --name MHQ_HTTP_SERVER diff --git a/web-server/scripts/setup-env.sh b/web-server/scripts/setup-env.sh index b6e89b50b..79f6518a4 100755 --- a/web-server/scripts/setup-env.sh +++ b/web-server/scripts/setup-env.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh set -e DIR=$(dirname $0) diff --git a/web-server/scripts/utils.sh b/web-server/scripts/utils.sh index e76232bd7..f7daf7836 100755 --- a/web-server/scripts/utils.sh +++ b/web-server/scripts/utils.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh set -e function catch_force_exit() { @@ -29,4 +29,4 @@ function install_yarn_cmd_if_not_exists() { else echo "$1 command exists" fi -} \ No newline at end of file +} diff --git a/web-server/scripts/zip.sh b/web-server/scripts/zip.sh index 8fc8694b5..ea49dd6cb 100755 --- a/web-server/scripts/zip.sh +++ b/web-server/scripts/zip.sh @@ -1,4 +1,4 @@ -#! /bin/bash +#! /bin/sh set -e DIR=$(dirname $0) @@ -17,11 +17,8 @@ tar -czf \ http-server.js \ .next \ next.config.js \ - .env* \ public \ - sentry.* \ scripts \ - config -echo "Completed artifact creation" \ No newline at end of file +echo "Completed artifact creation"