Dx improvements

.gitignore

@@ -194,6 +194,3 @@ tsconfig.tsbuildinfo
 
 # Sentry
 .sentryclirc
-
-# config files
-./apiserver/dora/config/config.ini

Dockerfile

@@ -27,7 +27,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 FROM python:3.9-slim
 
 ENV DB_HOST=localhost
-ENV DB_NAME=dora-oss
+ENV DB_NAME=mhq-oss
 ENV DB_PASS=postgres
 ENV DB_PORT=5434
 ENV DB_USER=postgres

Dockerfile.dev

@@ -6,90 +6,10 @@ ARG BACKEND_ENABLED=true
 ARG FRONTEND_ENABLED=true
 ARG CRON_ENABLED=true
 
-# Build the backend
-FROM python:3.9-slim as backend-build
-
-# Prevents Python from writing pyc files.
-ENV PYTHONDONTWRITEBYTECODE=1
-
-WORKDIR /app/
-COPY ./backend /app/backend
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        gcc \
-        libpq-dev \
-        build-essential \
-    && cd ./backend/ \
-    && python3 -m venv /opt/venv \
-    && /opt/venv/bin/pip install --upgrade pip \
-    && /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt
-
-# Final image
-FROM python:3.9-slim
-
-WORKDIR /app
-COPY --from=backend-build /opt/venv /opt/venv
-
-COPY . /app/
-
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-        gcc \
-        build-essential \
-        libpq-dev \
-        cron \
-        postgresql \
-        postgresql-contrib \
-        redis-server \
-        supervisor \
-        curl \
-    && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
-    && apt-get install -y nodejs \
-    && mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob \
-    && chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh \
-    && mv ./setup_utils/supervisord-dev.conf /etc/supervisord.conf \
-    && mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ \
-    && echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/15/main/pg_hba.conf \
-    && echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf \
-    && npm install --global yarn --force \
-    && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \
-    && chmod +x /usr/local/bin/dbmate \
-    && mkdir -p /var/log/postgres \
-    && touch /var/log/postgres/postgres.log \
-    && mkdir -p /var/log/init_db \
-    && touch /var/log/init_db/init_db.log \
-    && mkdir -p /var/log/redis \
-    && touch /var/log/redis/redis.log \
-    && mkdir -p /var/log/apiserver \
-    && touch /var/log/apiserver/apiserver.log \
-    && mkdir -p /var/log/webserver \
-    && touch /var/log/webserver/webserver.log \
-    && mkdir -p /var/log/cron \
-    && touch /var/log/cron/cron.log \
-    && chmod 0644 /etc/cron.d/cronjob \
-    && crontab /etc/cron.d/cronjob \
-    && /app/setup_utils/generate_config_ini.sh -t /app/backend/analytics_server/mhq/config \
-    && cd /app/web-server \
-    && yarn \
-    && rm -rf ./artifacts \
-    && cd /app/ \
-    && tar cfz web-server.tar.gz ./web-server \
-    && rm -rf ./web-server && mkdir -p /app/web-server \
-    && tar cfz /opt/venv.tar.gz /opt/venv/ \
-    && rm -rf /opt/venv && mkdir -p /opt/venv \
-    && yarn cache clean \
-    && apt-get clean \
-    && rm -rf /var/lib/apt/lists/*
-
-ENV POSTGRES_DB_ENABLED=true
-ENV DB_INIT_ENABLED=true
-ENV REDIS_ENABLED=true
-ENV BACKEND_ENABLED=true
-ENV FRONTEND_ENABLED=true
-ENV CRON_ENABLED=true
-ENV ENVIRONMENT=dev
+FROM middlewareeng/oss-base:latest
 
 ENV DB_HOST=localhost
-ENV DB_NAME=dora-oss
+ENV DB_NAME=mhq-oss
 ENV DB_PASS=postgres
 ENV DB_PORT=5434
 ENV DB_USER=postgres
@@ -104,6 +24,48 @@ ENV INTERNAL_SYNC_API_BASE_URL=http://localhost:9697
 ENV NEXT_PUBLIC_APP_ENVIRONMENT="prod"
 ENV PATH="/opt/venv/bin:/usr/lib/postgresql/15/bin:/usr/local/bin:$PATH"
 
-EXPOSE 3333 
+WORKDIR /app
+COPY . /app/
+
+WORKDIR /app/backend
+RUN python3 -m venv /opt/venv
+RUN /opt/venv/bin/pip install --upgrade pip
+RUN /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt
+
+WORKDIR /app
+RUN mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob
+RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh
+RUN mv /app/setup_utils/supervisord-dev.conf /etc/supervisord.conf
+RUN mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/
+RUN echo "host all  all    0.0.0.0/0  md5" >> /etc/postgresql/15/main/pg_hba.conf
+RUN echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf
+RUN sed -i "s/^port = .*/port = ${DB_PORT}/" /etc/postgresql/15/main/postgresql.conf
+RUN npm install --global yarn --force
+RUN mkdir -p /var/log/postgres
+RUN touch /var/log/postgres/postgres.log
+RUN mkdir -p /var/log/init_db
+RUN touch /var/log/init_db/init_db.log
+RUN mkdir -p /var/log/redis
+RUN touch /var/log/redis/redis.log
+RUN mkdir -p /var/log/apiserver
+RUN touch /var/log/apiserver/apiserver.log
+RUN mkdir -p /var/log/webserver
+RUN touch /var/log/webserver/webserver.log
+RUN mkdir -p /var/log/cron
+RUN touch /var/log/cron/cron.log
+RUN chmod 0644 /etc/cron.d/cronjob
+RUN crontab /etc/cron.d/cronjob
+RUN /app/setup_utils/generate_config_ini.sh -t /app/backend/analytics_server/mhq/config
+
+WORKDIR /app/web-server
+RUN yarn install --verbose
+
+ENV POSTGRES_DB_ENABLED=true
+ENV DB_INIT_ENABLED=true
+ENV REDIS_ENABLED=true
+ENV BACKEND_ENABLED=true
+ENV FRONTEND_ENABLED=true
+ENV CRON_ENABLED=true
+ENV ENVIRONMENT=dev
 
 CMD ["/bin/bash", "-c", "/app/setup_utils/start.sh"]

Dockerfile.prebuilt

@@ -0,0 +1,19 @@
+FROM python:3.9-slim as oss-base
+
+# Install necessary packages for building the backend
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        gcc \
+        build-essential \
+        libpq-dev \
+        cron \
+        postgresql \
+        postgresql-contrib \
+        redis-server \
+        supervisor \
+        curl \
+    && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
+    && apt-get install -y nodejs \
+    && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \
+    && chmod +x /usr/local/bin/dbmate \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*

cli/source/app.tsx

@@ -1,8 +1,16 @@
-import chalk from 'chalk';
 import { Box, Newline, Static, Text, useApp, useInput } from 'ink';
 import Spinner from 'ink-spinner';
 import { splitEvery } from 'ramda';
-import { ReactNode, useCallback, useEffect, useMemo, useState } from 'react';
+import {
+  ReactNode,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState
+} from 'react';
+
+import { ChildProcessWithoutNullStreams } from 'child_process';
 
 import {
   AppStates,
@@ -32,6 +40,8 @@ const CliUi = () => {
 
   const lineLimit = getLineLimit();
 
+  const processRef = useRef<ChildProcessWithoutNullStreams | null>();
+
   const runCommandOpts = useMemo<Parameters<typeof runCommand>['2']>(
     () => ({
       onData: (line) =>
@@ -84,15 +94,13 @@ const CliUi = () => {
     }
 
     if (input === 'x') {
+      if (!processRef.current) return;
+
       setAppState(AppStates.TEARDOWN);
-      runCommand('docker-compose', ['down'], runCommandOpts)
-        .promise.catch((err) => {
-          addLog(chalk.red(`Error logs: ${err.message}`));
-        })
-        .finally(() => {
-          setAppState(AppStates.TERMINATED);
-          setTimeout(() => exit(), 500);
-        });
+      processRef.current.kill();
+      processRef.current.stdout.destroy();
+      processRef.current.stderr.destroy();
+      exit();
     }
   });
 
@@ -101,20 +109,25 @@ const CliUi = () => {
       runCommand('docker-compose', ['down'], runCommandOpts);
     };
 
-    runCommand('docker-compose', ['watch'], runCommandOpts).process?.stdout.on(
-      'data',
-      (data) => {
-        let watch_logs = String(data);
-        if (watch_logs.includes(READY_MESSAGES[LogSource.DockerWatch])) {
-          addLog('\n🚀 Container ready 🚀\n');
-          setAppState(AppStates.DOCKER_READY);
-        }
+    const process = runCommand(
+      'docker-compose',
+      ['watch'],
+      runCommandOpts
+    ).process;
+
+    processRef.current = process;
+
+    process?.stdout.on('data', (data) => {
+      let watch_logs = String(data);
+      if (watch_logs.includes(READY_MESSAGES[LogSource.DockerWatch])) {
+        addLog('\n🚀 Container ready 🚀\n');
+        setAppState(AppStates.DOCKER_READY);
       }
-    );
+    });
 
-    process.on('exit', listener);
+    globalThis.process.on('exit', listener);
     return () => {
-      process.off('exit', listener);
+      globalThis.process.off('exit', listener);
     };
   }, [addLog, runCommandOpts, setAppState]);
 
@@ -140,7 +153,7 @@ const CliUi = () => {
               case AppStates.INIT:
                 return (
                   <Text color="blue">
-                    Status: Preparing docker container...{' '}
+                    Status: Preparing docker container... [Press X to abort]{' '}
                     <Text bold color="yellow">
                       <Spinner type="material" />
                     </Text>

cli/source/hooks/useLogs.tsx

@@ -104,7 +104,7 @@ export const useLogs = (
         'docker',
         [
           'exec',
-          'dora-metrics',
+          'middleware-dev',
           '/bin/bash',
           '-c',
           `tail -f /var/log/${logFile}.log`

dev.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 cd ./cli
-{yarn && yarn build} > /dev/null 2>&1
+{ yarn install && yarn build; } > /dev/null 2>&1
 yarn start
 

docker-compose.yml

@@ -19,8 +19,8 @@ services:
     ports:
       - "9696:9696"
       - "9697:9697"
-      - "3005:3333"
-      - "5436:5434"
+      - "3333:3333"
+      - "5434:5434"
       - "6379:6379"
 
     extra_hosts:
@@ -29,39 +29,35 @@ services:
     develop:
       watch:
         - action: sync
-          path: ./apiserver
-          target: /app/backend/apiserver
+          path: ./backend/analytics_server
+          target: /app/backend/analytics_server
           ignore: 
             - venv
             - __pycache__
             - env.example
 
         - action: rebuild
-          path: ./apiserver/requirements.txt
+          path: ./backend/requirements.txt
 
         - action: rebuild
-          path: ./apiserver/dev-requirements.txt
+          path: ./backend/dev-requirements.txt
 
         - action: rebuild
-          path: ./supervisord.conf
+          path: ./setup_util/supervisord.conf
 
         - action: rebuild
-          path: ./init_db.sh
+          path: ./setup_utils/init_db.sh
 
         - action: rebuild
           path: ./.env
 
         - action: sync+restart
-          path: ./apiserver/.env.local
-          target: /app/backend/apiserver/.env.local
-
-        - action: sync+restart
-          path: ./apiserver/dora/service/sync_data.py
-          target: /app/backend/apiserver/dora/service/sync_data.py
+          path: ./backend/analytics_server/.env.local
+          target: /app/backend/analytics_server/.env.local
 
         - action: sync
           path: ./web-server
-          target: /app/backend/web-server
+          target: /app/web-server
           ignore: 
             - ./web-server/.vscode 
             - ./web-server/node_modules

setup_utils/start.sh

@@ -25,10 +25,4 @@ if [ "$ENVIRONMENT" != "dev" ]; then
     yarn build
 fi
 
-if [ "$ENVIRONMENT" = "dev" ]; then
-    SUPERVISOR_CONF="/etc/supervisord-dev.conf"
-else
-    SUPERVISOR_CONF="/etc/supervisord.conf"
-fi
-
-/usr/bin/supervisord -c "$SUPERVISOR_CONF"
+/usr/bin/supervisord -c "/etc/supervisord.conf"

setup_utils/supervisord-dev.conf

@@ -39,7 +39,7 @@ autostart=%(ENV_REDIS_ENABLED)s
 
 [program:backend_sync]
 priority=5
-command=/bin/bash -c "/opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9697 --reload sync_app:app"
+command=/bin/bash -c "/opt/venv/bin/gunicorn -w 1 -b 0.0.0.0:9697 --timeout 0  --reload sync_app:app"
 directory=/app/backend/analytics_server
 stdout_logfile=/var/log/apiserver/apiserver.log
 stdout_logfile_maxbytes=512KB
@@ -66,7 +66,7 @@ environment=BACKEND_ENABLED=%(ENV_BACKEND_ENABLED)s
 autostart=%(ENV_BACKEND_ENABLED)s
 
 [program:frontend]
-command=/bin/bash -c "yarn dev"
+command=/bin/bash -c "source ~/.bashrc && yarn dev"
 directory=/app/web-server
 stdout_logfile=/var/log/webserver/webserver.log
 stdout_logfile_maxbytes=512KB

setup_utils/supervisord.conf

@@ -39,7 +39,7 @@ autostart=%(ENV_REDIS_ENABLED)s
 
 [program:backend_sync]
 priority=5
-command=/bin/bash -c "/opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9697 --reload sync_app:app"
+command=/bin/bash -c "/opt/venv/bin/gunicorn -w 2 -b 0.0.0.0:9697 --timeout 0  --reload sync_app:app"
 directory=/app/backend/analytics_server
 stdout_logfile=/var/log/apiserver/apiserver.log
 stdout_logfile_maxbytes=512KB

web-server/package.json

@@ -84,7 +84,7 @@
     "prod-tun": "bash -c \"$PROD_TUNNEL\"",
     "stage-tun": "bash -c \"$STAGE_TUNNEL\"",
     "http": "node http-server",
-    "dev": "NEXT_MANUAL_SIG_HANDLE=true yarn env && next",
+    "dev": "NEXT_MANUAL_SIG_HANDLE=true next",
     "dev-prod": "NEXT_MANUAL_SIG_HANDLE=true yarn env prod && next",
     "build": "./scripts/build.sh",
     "zip": "./scripts/zip.sh",