multi-arch build support

.dockerignore

@@ -5,3 +5,14 @@
 ./*/env.example
 ./*/.vscode
 ./*/node_modules
+.gitignore
+./CONTRIBUTING.md
+./docker-compose.yml
+./*/docker-compose.yml
+./Dockerfile
+./Dockerfile.dev
+./LICENSE
+./README.md
+./*/README.*
+./dev.sh
+./setup_utils/install.sh

.github/workflows/build.yml

@@ -33,7 +33,17 @@ jobs:
         uses: aws-actions/amazon-ecr-login@v2
         with:
           registry-type: public
-
+
+      - name: Set up Docker
+        uses: crazy-max/ghaction-setup-docker@v3
+        with:
+          daemon-config: |
+            {
+              "debug": true,
+              "features": {
+                "containerd-snapshotter": true
+              }
+            }
       - name: Build Docker image
         run: docker build --tag middleware:latest --platform linux/amd64,linux/arm64 .
 

Dockerfile

@@ -1,108 +1,115 @@
+ARG ENVIRONMENT=prod
+ARG POSTGRES_DB_ENABLED=true
+ARG DB_INIT_ENABLED=true
+ARG REDIS_ENABLED=true
+ARG BACKEND_ENABLED=true
+ARG FRONTEND_ENABLED=true
+ARG CRON_ENABLED=true
+
 # Build the backend
-FROM python:3.9-slim as backend-build
+FROM python:3.9.19-alpine3.19 as backend-build
 
 # Prevents Python from writing pyc files.
 ENV PYTHONDONTWRITEBYTECODE=1
 
-ARG ENVIRONMENT=dev
+WORKDIR /app/
+COPY ./apiserver /app/apiserver
+RUN apk update && \
+    apk add --no-cache \
+        git \
+        postgresql-dev \
+        gcc \
+        musl-dev \
+        python3-dev \
+    && cd ./apiserver \
+    && python3 -m venv /opt/venv \
+    && /opt/venv/bin/pip install --upgrade pip \
+    && /opt/venv/bin/pip install -r requirements.txt
 
-WORKDIR /app/backend
-COPY ./apiserver /app/backend/apiserver
-RUN apt-get update && apt-get install -y --no-install-recommends \
-  git \
-  libpq-dev \
-  gcc \
-  build-essential \
-  && cd ./apiserver \
-  && python3 -m venv /opt/venv \
-  && /opt/venv/bin/pip install --upgrade pip \
-  && if [ "$ENVIRONMENT" = "dev" ]; then \
-  /opt/venv/bin/pip install -r dev-requirements.txt -r requirements.txt; \
-  fi
-
-# Build the frontend
-FROM node:16-alpine as frontend-build
-WORKDIR /app/frontend/
-COPY ./web-server /app/frontend/web-server
-RUN apk add --no-cache yarn && \
-    cd web-server && \
-   if [ "$ENVIRONMENT" = "dev" ]; then \
-    yarn install && yarn cache clean \
-  else \
-    yarn install && \
-    yarn build && \
-    yarn cache clean && \
-    tar -xvf  ./artifacts/artifact.tar.gz --directory=./artifacts && \
-    find ./ -mindepth 1 -maxdepth 1 ! -name 'artifacts' -exec rm -rf {} + \
-    mv ./artifacts/* ./* && rm -rf ./artifacts; \
-  fi
+FROM node:16.20.2-alpine3.18 as node
 
 # Final image
-FROM python:3.9-slim
+FROM postgres:alpine3.19
 
 WORKDIR /app
 COPY --from=backend-build /opt/venv /opt/venv
-COPY --from=backend-build /app/backend /app/backend
-COPY --from=frontend-build /app/frontend/web-server/ /app/frontend/web-server
+COPY --from=backend-build /usr/local/bin/ /usr/local/bin/
+COPY --from=backend-build /usr/local/lib/ /usr/local/lib/
+COPY --from=backend-build /usr/local/include /usr/local/include
 
-COPY ./database-docker/db/ /app/db/
-COPY ./init_db.sh /app/
-COPY ./generate_config_ini.sh /app/
-COPY ./cronjob.txt /etc/cron.d/cronjob
-COPY ./supervisord.conf /etc/supervisord.conf
+COPY --from=node /usr/lib /usr/lib
+COPY --from=node /usr/local/lib /usr/local/lib
+COPY --from=node /usr/local/include /usr/local/include
+COPY --from=node /usr/local/bin /usr/local/bin
 
-RUN chmod +x /app/init_db.sh \
-  && apt-get update && apt-get install -y --no-install-recommends \
-  gcc \
-  build-essential \
-  libpq-dev \
-  cron \
-  postgresql \
-  postgresql-contrib \
-  redis-server \
-  supervisor \
-  curl \
-  && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
-  && apt-get install -y nodejs \
-  && npm install --global yarn \
-  && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \
-  && chmod +x /usr/local/bin/dbmate \
-  && echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/15/main/pg_hba.conf \
-  && echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf \
-  && mkdir -p /var/log/postgres \
-  && touch /var/log/postgres/postgres.log \
-  && mkdir -p /var/log/init_db \
-  && touch /var/log/init_db/init_db.log \
-  && mkdir -p /var/log/redis \
-  && touch /var/log/redis/redis.log \
-  && mkdir -p /var/log/apiserver \
-  && touch /var/log/apiserver/apiserver.log \
-  && mkdir -p /var/log/webserver \
-  && touch /var/log/webserver/webserver.log \
-  && mkdir -p /var/log/cron \
-  && touch /var/log/cron/cron.log \
-  && chmod 0644 /etc/cron.d/cronjob \
-  && crontab /etc/cron.d/cronjob \
-  && yarn cache clean \
-  && apt-get clean \
-  && rm -rf /var/lib/apt/lists/* \
-  && /app/generate_config_ini.sh -t /app/backend/apiserver/dora/config
+COPY . /app/
 
-ARG POSTGRES_DB_ENABLED=true
-ARG DB_INIT_ENABLED=true
-ARG REDIS_ENABLED=true
-ARG BACKEND_ENABLED=true
-ARG FRONTEND_ENABLED=true
-ARG CRON_ENABLED=true
+RUN apk add --no-cache \
+        gcc \
+        build-base \
+        libpq \
+        postgresql-dev \
+        postgresql-client \
+        postgresql-contrib \
+        redis \
+        supervisor \
+        curl \
+    && mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob \
+    && chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh \
+    && mv ./setup_utils/supervisord.conf /etc/supervisord.conf \
+    && mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ \
+    && su - postgres -c "initdb -D /var/lib/postgresql/data" \
+    && su - postgres -c "echo \"host all  all    0.0.0.0/0  md5\" >> /var/lib/postgresql/data/pg_hba.conf" \
+    && su - postgres -c "echo \"listen_addresses='*'\" >> /var/lib/postgresql/data/postgresql.conf" \
+    && su - postgres -c "rm -f /var/lib/postgresql/data/postmaster.pid" \
+    && npm install --global yarn --force \
+    && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \
+    && chmod +x /usr/local/bin/dbmate \
+    && mkdir -p /var/log/postgres \
+    && touch /var/log/postgres/postgres.log \
+    && mkdir -p /var/log/init_db \
+    && touch /var/log/init_db/init_db.log \
+    && mkdir -p /var/log/redis \
+    && touch /var/log/redis/redis.log \
+    && mkdir -p /var/log/apiserver \
+    && touch /var/log/apiserver/apiserver.log \
+    && mkdir -p /var/log/webserver \
+    && touch /var/log/webserver/webserver.log \
+    && mkdir -p /var/log/cron \
+    && touch /var/log/cron/cron.log \
+    && chmod 0644 /etc/cron.d/cronjob \
+    && crontab /etc/cron.d/cronjob \
+    && /app/setup_utils/generate_config_ini.sh -t /app/apiserver/dora/config \
+    && cd /app/web-server \
+    && yarn && yarn build \
+    && rm -rf ./artifacts \
+    && cd /app/ \
+    && tar cfz web-server.tar.gz ./web-server \
+    && rm -rf ./web-server && mkdir -p /app/web-server \
+    && tar cfz /opt/venv.tar.gz /opt/venv/ \
+    && rm -rf /opt/venv && mkdir -p /opt/venv \
+    && yarn cache clean \
+    && rm -rf /var/cache/apk/*
 
 ENV POSTGRES_DB_ENABLED=$POSTGRES_DB_ENABLED
 ENV DB_INIT_ENABLED=$DB_INIT_ENABLED
 ENV REDIS_ENABLED=$REDIS_ENABLED
 ENV BACKEND_ENABLED=$BACKEND_ENABLED
 ENV FRONTEND_ENABLED=$FRONTEND_ENABLED
 ENV CRON_ENABLED=$CRON_ENABLED
-ENV PATH="/opt/venv/bin:/usr/lib/postgresql/15/bin:$PATH"
+ENV DB_HOST=localhost
+ENV DB_NAME=dora-oss
+ENV DB_PASS=postgres
+ENV DB_PORT=5432
+ENV DB_USER=postgres
+ENV REDIS_HOST=localhost
+ENV REDIS_PORT=6379
+ENV PORT=3000
+ENV NEXT_PUBLIC_APP_ENVIRONMENT="staging"
+ENV INTERNAL_API_BASE_URL=http://localhost:9696
+ENV NEXT_PUBLIC_APP_ENVIRONMENT="prod"
+ENV PATH="/opt/venv/bin:$PATH"
 
 EXPOSE 5432 6379 9696 3000
 
-CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]
+CMD ["/bin/sh", "-c", "/app/setup_utils/start.sh"]

docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   dora-metrics:
-    container_name: dora-metrics
+    container_name: middleware
     build:
       context: ./
       dockerfile: Dockerfile

generate_config_ini.sh → setup_utils/generate_config_ini.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 # Parse command line arguments
 while [[ $# -gt 0 ]]; do

init_db.sh → setup_utils/init_db.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 
 set -e
 set -u

setup_utils/start.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+echo 'MHQ_EXTRACT_BACKEND_DEPENDENCIES'
+if [ -f /opt/venv.tar.gz ]; then
+    mkdir -p /opt/venv
+    tar xzf /opt/venv.tar.gz -C /opt/venv --strip-components=2
+    rm -rf /opt/venv.tar.gz
+else
+    echo "Tar file /opt/venv.tar.gz does not exist. Skipping extraction."
+fi
+
+echo 'MHQ_EXTRACT_FRONTEND'
+if [ -f /app/web-server.tar.gz ]; then
+    mkdir -p /app/frontend/web-server
+    tar xzf /app/web-server.tar.gz -C /app/frontend/web-server --strip-components=2
+    rm -rf /app/web-server.tar.gz
+else
+    echo "Tar file /app/web-server.tar.gz does not exist. Skipping extraction."
+fi
+
+pg_ctl stop -D /var/lib/postgres/data;
+
+echo 'MHQ_STARTING SUPERVISOR'
+/usr/bin/supervisord -c /etc/supervisord.conf

supervisord.conf → setup_utils/supervisord.conf

@@ -3,19 +3,19 @@ nodaemon=true
 
 [program:postgres]
 user=postgres
-command=postgres -D /var/lib/postgresql/15/main -c config_file=/etc/postgresql/15/main/postgresql.conf
+command=/bin/sh -c "pg_ctl start -D /var/lib/postgresql/data"
 stdout_logfile=/var/log/postgres/postgres.log
 stdout_logfile_maxbytes=512KB
 stderr_logfile=/var/log/postgres/postgres.log
 stderr_logfile_maxbytes=512KB
-autorestart=true
+autorestart=false
 environment=POSTGRES_DB_ENABLED=%(ENV_POSTGRES_DB_ENABLED)s
 autostart=%(ENV_POSTGRES_DB_ENABLED)s
 
 [program:initialize_db]
-command=/app/init_db.sh 
+command=/app/setup_utils/init_db.sh 
 directory=/app
-startsecs=0
+startsecs=5
 stdout_logfile=/var/log/init_db/init_db.log
 stdout_logfile_maxbytes=512KB
 stderr_logfile=/var/log/init_db/init_db.log
@@ -35,8 +35,8 @@ environment=REDIS_ENABLED=%(ENV_REDIS_ENABLED)s
 autostart=%(ENV_REDIS_ENABLED)s
 
 [program:backend]
-command=/bin/bash -c "/opt/venv/bin/pip install --upgrade pip && /opt/venv/bin/pip install -r requirements.txt && /opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9696 --reload app:app"
-directory=/app/backend/apiserver
+command=/bin/sh -c "/opt/venv/bin/pip install --upgrade pip && /opt/venv/bin/pip install -r requirements.txt && /opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9696 --reload app:app"
+directory=/app/apiserver
 stdout_logfile=/var/log/apiserver/apiserver.log
 stdout_logfile_maxbytes=512KB
 stderr_logfile=/var/log/apiserver/apiserver.log
@@ -48,8 +48,8 @@ environment=BACKEND_ENABLED=%(ENV_BACKEND_ENABLED)s
 autostart=%(ENV_BACKEND_ENABLED)s
 
 [program:frontend]
-command=/bin/bash -c "yarn && yarn cache clean && yarn http"
-directory=/app/frontend/web-server
+command=/bin/sh -c "yarn http"
+directory=/app/web-server
 stdout_logfile=/var/log/webserver/webserver.log
 stdout_logfile_maxbytes=512KB
 stderr_logfile=/var/log/webserver/webserver.log
@@ -61,7 +61,7 @@ environment=FRONTEND_ENABLED=%(ENV_FRONTEND_ENABLED)s
 autostart=%(ENV_FRONTEND_ENABLED)s
 
 [program:cron]
-command=/usr/sbin/cron -f -l 1
+command=/usr/sbin/crond -f -l 1
 autorestart=true
 startsecs=10
 stderr_logfile=/var/log/cron.log

web-server/scripts/build.sh

@@ -1,4 +1,4 @@
-#! /bin/bash
+#! /bin/sh
 set -e
 
 DIR=$(dirname $0)
@@ -13,4 +13,4 @@ yarn run next build
 echo "EXITED $?"
 
 rm -rf .next/cache
-yarn run zip
+yarn run zip

web-server/scripts/server-init.sh

@@ -1,4 +1,4 @@
-#! /bin/bash
+#! /bin/sh
 set -e
 
 DIR=$(dirname $0)
@@ -14,4 +14,4 @@ pm2 delete MHQ_HTTP_SERVER
 set -e
 
 NEXT_MANUAL_SIG_HANDLE=true
-pm2 start "yarn http" --name MHQ_HTTP_SERVER
+pm2 start "yarn http" --name MHQ_HTTP_SERVER

web-server/scripts/setup-env.sh

@@ -1,4 +1,4 @@
-#! /bin/bash
+#! /bin/sh
 set -e
 
 DIR=$(dirname $0)

web-server/scripts/utils.sh

@@ -1,4 +1,4 @@
-#! /bin/bash
+#! /bin/sh
 set -e
 
 function catch_force_exit() {
@@ -29,4 +29,4 @@ function install_yarn_cmd_if_not_exists() {
   else
     echo "$1 command exists"
   fi
-}
+}

web-server/scripts/zip.sh

@@ -1,4 +1,4 @@
-#! /bin/bash
+#! /bin/sh
 set -e
 
 DIR=$(dirname $0)
@@ -17,11 +17,8 @@ tar -czf \
   http-server.js \
   .next \
   next.config.js \
-  .env* \
   public \
-  sentry.* \
   scripts \
-  config
 
 
-echo "Completed artifact creation"
+echo "Completed artifact creation"