From 8fe5b0870a2d64e4d1c3bec34ec2f29e7ced4977 Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 13:04:59 +0530 Subject: [PATCH 01/14] Change container name used for logs and fix process exit handler --- cli/source/app.tsx | 57 ++++++++++++++++++++++-------------- cli/source/hooks/useLogs.tsx | 2 +- 2 files changed, 36 insertions(+), 23 deletions(-) diff --git a/cli/source/app.tsx b/cli/source/app.tsx index d957b194c..aaa936e44 100644 --- a/cli/source/app.tsx +++ b/cli/source/app.tsx @@ -1,8 +1,16 @@ -import chalk from 'chalk'; import { Box, Newline, Static, Text, useApp, useInput } from 'ink'; import Spinner from 'ink-spinner'; import { splitEvery } from 'ramda'; -import { ReactNode, useCallback, useEffect, useMemo, useState } from 'react'; +import { + ReactNode, + useCallback, + useEffect, + useMemo, + useRef, + useState +} from 'react'; + +import { ChildProcessWithoutNullStreams } from 'child_process'; import { AppStates, @@ -32,6 +40,8 @@ const CliUi = () => { const lineLimit = getLineLimit(); + const processRef = useRef(); + const runCommandOpts = useMemo['2']>( () => ({ onData: (line) => @@ -84,15 +94,13 @@ const CliUi = () => { } if (input === 'x') { + if (!processRef.current) return; + setAppState(AppStates.TEARDOWN); - runCommand('docker-compose', ['down'], runCommandOpts) - .promise.catch((err) => { - addLog(chalk.red(`Error logs: ${err.message}`)); - }) - .finally(() => { - setAppState(AppStates.TERMINATED); - setTimeout(() => exit(), 500); - }); + processRef.current.kill(); + processRef.current.stdout.destroy(); + processRef.current.stderr.destroy(); + exit(); } }); @@ -101,20 +109,25 @@ const CliUi = () => { runCommand('docker-compose', ['down'], runCommandOpts); }; - runCommand('docker-compose', ['watch'], runCommandOpts).process?.stdout.on( - 'data', - (data) => { - let watch_logs = String(data); - if (watch_logs.includes(READY_MESSAGES[LogSource.DockerWatch])) { - addLog('\nšŸš€ Container ready šŸš€\n'); - setAppState(AppStates.DOCKER_READY); - } + const process = runCommand( + 'docker-compose', + ['watch'], + runCommandOpts + ).process; + + processRef.current = process; + + process?.stdout.on('data', (data) => { + let watch_logs = String(data); + if (watch_logs.includes(READY_MESSAGES[LogSource.DockerWatch])) { + addLog('\nšŸš€ Container ready šŸš€\n'); + setAppState(AppStates.DOCKER_READY); } - ); + }); - process.on('exit', listener); + globalThis.process.on('exit', listener); return () => { - process.off('exit', listener); + globalThis.process.off('exit', listener); }; }, [addLog, runCommandOpts, setAppState]); @@ -140,7 +153,7 @@ const CliUi = () => { case AppStates.INIT: return ( - Status: Preparing docker container...{' '} + Status: Preparing docker container... [Press X to abort]{' '} diff --git a/cli/source/hooks/useLogs.tsx b/cli/source/hooks/useLogs.tsx index 6013b9b5f..62b650264 100644 --- a/cli/source/hooks/useLogs.tsx +++ b/cli/source/hooks/useLogs.tsx @@ -104,7 +104,7 @@ export const useLogs = ( 'docker', [ 'exec', - 'dora-metrics', + 'middleware-dev', '/bin/bash', '-c', `tail -f /var/log/${logFile}.log` From 616280bb424acc85fbe22abf43f7c066632d4412 Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 13:06:07 +0530 Subject: [PATCH 02/14] Fix syntax for running cli via yarn commands in dev.sh --- dev.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev.sh b/dev.sh index d2e2c47aa..ce76ac69f 100755 --- a/dev.sh +++ b/dev.sh @@ -1,6 +1,6 @@ #!/bin/bash cd ./cli -{yarn && yarn build} > /dev/null 2>&1 +{ yarn install && yarn build; } > /dev/null 2>&1 yarn start From 868b2a629847a28e835aa90d0926d37ff4561ecf Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 13:06:31 +0530 Subject: [PATCH 03/14] Remove commands that deleted/cleared caches from Dockerfile.dev --- Dockerfile.dev | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index a383c01a1..2a9d67c44 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -69,16 +69,7 @@ RUN apt-get update && \ && crontab /etc/cron.d/cronjob \ && /app/setup_utils/generate_config_ini.sh -t /app/backend/analytics_server/mhq/config \ && cd /app/web-server \ - && yarn \ - && rm -rf ./artifacts \ - && cd /app/ \ - && tar cfz web-server.tar.gz ./web-server \ - && rm -rf ./web-server && mkdir -p /app/web-server \ - && tar cfz /opt/venv.tar.gz /opt/venv/ \ - && rm -rf /opt/venv && mkdir -p /opt/venv \ - && yarn cache clean \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* + && yarn install ENV POSTGRES_DB_ENABLED=true ENV DB_INIT_ENABLED=true From 30becd9d150f300c711837aabbd8f8137d6a853e Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 13:33:43 +0530 Subject: [PATCH 04/14] Remove yarn env related commands and files. --- web-server/package.json | 2 +- web-server/scripts/setup-env.sh | 31 ------------------------------- 2 files changed, 1 insertion(+), 32 deletions(-) delete mode 100755 web-server/scripts/setup-env.sh diff --git a/web-server/package.json b/web-server/package.json index 62a4e1da8..27a9596a6 100644 --- a/web-server/package.json +++ b/web-server/package.json @@ -84,7 +84,7 @@ "prod-tun": "bash -c \"$PROD_TUNNEL\"", "stage-tun": "bash -c \"$STAGE_TUNNEL\"", "http": "node http-server", - "dev": "NEXT_MANUAL_SIG_HANDLE=true yarn env && next", + "dev": "NEXT_MANUAL_SIG_HANDLE=true next", "dev-prod": "NEXT_MANUAL_SIG_HANDLE=true yarn env prod && next", "build": "./scripts/build.sh", "zip": "./scripts/zip.sh", diff --git a/web-server/scripts/setup-env.sh b/web-server/scripts/setup-env.sh deleted file mode 100755 index b6e89b50b..000000000 --- a/web-server/scripts/setup-env.sh +++ /dev/null @@ -1,31 +0,0 @@ -#! /bin/bash -set -e - -DIR=$(dirname $0) - -source $DIR/utils.sh -is_project_root - -COMMON_ENV=$(cat ./.local_envs/.env.common 2> /dev/null) -LOCAL_ENV=$(cat ./.local_envs/.env.local 2> /dev/null) -STAGE_ENV=$(cat ./.local_envs/.env.staging 2> /dev/null) - -if [ "$1" == "prod" ]; then - echo "# PRODUCTION ENV CONFIG. USE WITH CAUTION" > .env.local; -else - echo "# STAGING ENV CONFIG" > .env.local; -fi - -echo "$COMMON_ENV" >> .env.local; - -if [ "$1" == "prod" ]; then - PROD_ENV=$(cat ./.local_envs/.env.production 2> /dev/null) - echo "$PROD_ENV" >> .env.local; -else - echo "$STAGE_ENV" >> .env.local; -fi - -echo -e "\n\n# LOCAL CONFIG" >> .env.local; -echo "$LOCAL_ENV" >> .env.local; - -echo "šŸŽ‰ .env.local updated" From 727a4a89babed1589356e352fcbf29309bd2405d Mon Sep 17 00:00:00 2001 From: Adnan Hashmi Date: Wed, 24 Apr 2024 13:50:44 +0530 Subject: [PATCH 05/14] compose changes to accomodate directory structure --- docker-compose.yml | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1947c16c7..2c45741c8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,8 +19,8 @@ services: ports: - "9696:9696" - "9697:9697" - - "3005:3333" - - "5436:5434" + - "3333:3333" + - "5434:5434" - "6379:6379" extra_hosts: @@ -29,39 +29,35 @@ services: develop: watch: - action: sync - path: ./apiserver - target: /app/backend/apiserver + path: ./backend/analytics_server + target: /app/backend/analytics_server ignore: - venv - __pycache__ - env.example - action: rebuild - path: ./apiserver/requirements.txt + path: ./backend/requirements.txt - action: rebuild - path: ./apiserver/dev-requirements.txt + path: ./backend/dev-requirements.txt - action: rebuild - path: ./supervisord.conf + path: ./setup_util/supervisord.conf - action: rebuild - path: ./init_db.sh + path: ./setup_utils/init_db.sh - action: rebuild path: ./.env - action: sync+restart - path: ./apiserver/.env.local - target: /app/backend/apiserver/.env.local - - - action: sync+restart - path: ./apiserver/dora/service/sync_data.py - target: /app/backend/apiserver/dora/service/sync_data.py + path: ./backend/analytics_server/.env.local + target: /app/backend/analytics_server/.env.local - action: sync path: ./web-server - target: /app/backend/web-server + target: /app/web-server ignore: - ./web-server/.vscode - ./web-server/node_modules From 4c3fd73e2154da2e3d85e027d995af147190de1e Mon Sep 17 00:00:00 2001 From: Adnan Hashmi Date: Wed, 24 Apr 2024 14:56:23 +0530 Subject: [PATCH 06/14] make dockerfile.dev working --- Dockerfile.dev | 32 +++++++++++++++++--------------- setup_utils/start.sh | 8 +------- 2 files changed, 18 insertions(+), 22 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 2a9d67c44..89fbef875 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -26,6 +26,22 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Final image FROM python:3.9-slim +ENV DB_HOST=localhost +ENV DB_NAME=dora-oss +ENV DB_PASS=postgres +ENV DB_PORT=5434 +ENV DB_USER=postgres +ENV REDIS_HOST=localhost +ENV REDIS_PORT=6379 +ENV PORT=3333 +ENV SYNC_SERVER_PORT=9696 +ENV ANALYTICS_SERVER_PORT=9697 +ENV NEXT_PUBLIC_APP_ENVIRONMENT="staging" +ENV INTERNAL_API_BASE_URL=http://localhost:9696 +ENV INTERNAL_SYNC_API_BASE_URL=http://localhost:9697 +ENV NEXT_PUBLIC_APP_ENVIRONMENT="prod" +ENV PATH="/opt/venv/bin:/usr/lib/postgresql/15/bin:/usr/local/bin:$PATH" + WORKDIR /app COPY --from=backend-build /opt/venv /opt/venv @@ -50,6 +66,7 @@ RUN apt-get update && \ && mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ \ && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/15/main/pg_hba.conf \ && echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf \ + && sed -i "s/^port = .*/port = ${DB_PORT}/" /etc/postgresql/15/main/postgresql.conf \ && npm install --global yarn --force \ && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \ && chmod +x /usr/local/bin/dbmate \ @@ -79,21 +96,6 @@ ENV FRONTEND_ENABLED=true ENV CRON_ENABLED=true ENV ENVIRONMENT=dev -ENV DB_HOST=localhost -ENV DB_NAME=dora-oss -ENV DB_PASS=postgres -ENV DB_PORT=5434 -ENV DB_USER=postgres -ENV REDIS_HOST=localhost -ENV REDIS_PORT=6379 -ENV PORT=3333 -ENV SYNC_SERVER_PORT=9696 -ENV ANALYTICS_SERVER_PORT=9697 -ENV NEXT_PUBLIC_APP_ENVIRONMENT="staging" -ENV INTERNAL_API_BASE_URL=http://localhost:9696 -ENV INTERNAL_SYNC_API_BASE_URL=http://localhost:9697 -ENV NEXT_PUBLIC_APP_ENVIRONMENT="prod" -ENV PATH="/opt/venv/bin:/usr/lib/postgresql/15/bin:/usr/local/bin:$PATH" EXPOSE 3333 diff --git a/setup_utils/start.sh b/setup_utils/start.sh index 9c1ff621e..980a546a0 100644 --- a/setup_utils/start.sh +++ b/setup_utils/start.sh @@ -25,10 +25,4 @@ if [ "$ENVIRONMENT" != "dev" ]; then yarn build fi -if [ "$ENVIRONMENT" = "dev" ]; then - SUPERVISOR_CONF="/etc/supervisord-dev.conf" -else - SUPERVISOR_CONF="/etc/supervisord.conf" -fi - -/usr/bin/supervisord -c "$SUPERVISOR_CONF" +/usr/bin/supervisord -c "/etc/supervisord.conf" From 627a06f550f13d5ff05525c04e271227194f8c5d Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 15:19:53 +0530 Subject: [PATCH 07/14] Break up dockerfile.dev into multiple run commands. --- Dockerfile.dev | 79 +++++++++++++++++++++++++------------------------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 89fbef875..8ae746f9d 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -14,14 +14,12 @@ ENV PYTHONDONTWRITEBYTECODE=1 WORKDIR /app/ COPY ./backend /app/backend -RUN apt-get update && apt-get install -y --no-install-recommends \ - gcc \ - libpq-dev \ - build-essential \ - && cd ./backend/ \ - && python3 -m venv /opt/venv \ - && /opt/venv/bin/pip install --upgrade pip \ - && /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt +RUN apt-get update && apt-get install -y --no-install-recommends gcc libpq-dev build-essential + +WORKDIR /app/backend +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip install --upgrade pip +RUN /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt # Final image FROM python:3.9-slim @@ -47,8 +45,8 @@ COPY --from=backend-build /opt/venv /opt/venv COPY . /app/ -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ +RUN apt-get update +RUN apt-get install -y --no-install-recommends \ gcc \ build-essential \ libpq-dev \ @@ -57,36 +55,37 @@ RUN apt-get update && \ postgresql-contrib \ redis-server \ supervisor \ - curl \ - && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ - && apt-get install -y nodejs \ - && mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob \ - && chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh \ - && mv ./setup_utils/supervisord-dev.conf /etc/supervisord.conf \ - && mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ \ - && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/15/main/pg_hba.conf \ - && echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf \ - && sed -i "s/^port = .*/port = ${DB_PORT}/" /etc/postgresql/15/main/postgresql.conf \ - && npm install --global yarn --force \ - && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \ - && chmod +x /usr/local/bin/dbmate \ - && mkdir -p /var/log/postgres \ - && touch /var/log/postgres/postgres.log \ - && mkdir -p /var/log/init_db \ - && touch /var/log/init_db/init_db.log \ - && mkdir -p /var/log/redis \ - && touch /var/log/redis/redis.log \ - && mkdir -p /var/log/apiserver \ - && touch /var/log/apiserver/apiserver.log \ - && mkdir -p /var/log/webserver \ - && touch /var/log/webserver/webserver.log \ - && mkdir -p /var/log/cron \ - && touch /var/log/cron/cron.log \ - && chmod 0644 /etc/cron.d/cronjob \ - && crontab /etc/cron.d/cronjob \ - && /app/setup_utils/generate_config_ini.sh -t /app/backend/analytics_server/mhq/config \ - && cd /app/web-server \ - && yarn install + curl +RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - +RUN apt-get install -y nodejs +RUN mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob +RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh +RUN mv ./setup_utils/supervisord-dev.conf /etc/supervisord.conf +RUN mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ +RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/15/main/pg_hba.conf +RUN echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf +RUN sed -i "s/^port = .*/port = ${DB_PORT}/" /etc/postgresql/15/main/postgresql.conf +RUN npm install --global yarn --force +RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 +RUN chmod +x /usr/local/bin/dbmate +RUN mkdir -p /var/log/postgres +RUN touch /var/log/postgres/postgres.log +RUN mkdir -p /var/log/init_db +RUN touch /var/log/init_db/init_db.log +RUN mkdir -p /var/log/redis +RUN touch /var/log/redis/redis.log +RUN mkdir -p /var/log/apiserver +RUN touch /var/log/apiserver/apiserver.log +RUN mkdir -p /var/log/webserver +RUN touch /var/log/webserver/webserver.log +RUN mkdir -p /var/log/cron +RUN touch /var/log/cron/cron.log +RUN chmod 0644 /etc/cron.d/cronjob +RUN crontab /etc/cron.d/cronjob +RUN /app/setup_utils/generate_config_ini.sh -t /app/backend/analytics_server/mhq/config + +WORKDIR /app/web-server +RUN yarn install ENV POSTGRES_DB_ENABLED=true ENV DB_INIT_ENABLED=true From b2d97f6b921899b95e18f402c327aeefc5c58520 Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 16:06:20 +0530 Subject: [PATCH 08/14] Make yarn install be --verbose --- Dockerfile.dev | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index 8ae746f9d..875dffb96 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -85,7 +85,7 @@ RUN crontab /etc/cron.d/cronjob RUN /app/setup_utils/generate_config_ini.sh -t /app/backend/analytics_server/mhq/config WORKDIR /app/web-server -RUN yarn install +RUN yarn install --verbose ENV POSTGRES_DB_ENABLED=true ENV DB_INIT_ENABLED=true From a69cd0522bc17555e4e2d84210a0b7e247a718d5 Mon Sep 17 00:00:00 2001 From: amoghjalan Date: Wed, 24 Apr 2024 16:47:07 +0530 Subject: [PATCH 09/14] Updates Dockerfile.dev splitting into ass-base --- Dockerfile | 2 +- Dockerfile.dev | 48 +++++++++------------------------------------ Dockerfile.prebuilt | 19 ++++++++++++++++++ 3 files changed, 29 insertions(+), 40 deletions(-) create mode 100644 Dockerfile.prebuilt diff --git a/Dockerfile b/Dockerfile index 414f223f9..5975ed48b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ FROM python:3.9-slim ENV DB_HOST=localhost -ENV DB_NAME=dora-oss +ENV DB_NAME=mhq-oss ENV DB_PASS=postgres ENV DB_PORT=5434 ENV DB_USER=postgres diff --git a/Dockerfile.dev b/Dockerfile.dev index 875dffb96..378ff7a4b 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -6,26 +6,10 @@ ARG BACKEND_ENABLED=true ARG FRONTEND_ENABLED=true ARG CRON_ENABLED=true -# Build the backend -FROM python:3.9-slim as backend-build - -# Prevents Python from writing pyc files. -ENV PYTHONDONTWRITEBYTECODE=1 - -WORKDIR /app/ -COPY ./backend /app/backend -RUN apt-get update && apt-get install -y --no-install-recommends gcc libpq-dev build-essential - -WORKDIR /app/backend -RUN python3 -m venv /opt/venv -RUN /opt/venv/bin/pip install --upgrade pip -RUN /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt - -# Final image -FROM python:3.9-slim +FROM middlewareeng/oss-base:latest ENV DB_HOST=localhost -ENV DB_NAME=dora-oss +ENV DB_NAME=mhq-oss ENV DB_PASS=postgres ENV DB_PORT=5434 ENV DB_USER=postgres @@ -41,33 +25,22 @@ ENV NEXT_PUBLIC_APP_ENVIRONMENT="prod" ENV PATH="/opt/venv/bin:/usr/lib/postgresql/15/bin:/usr/local/bin:$PATH" WORKDIR /app -COPY --from=backend-build /opt/venv /opt/venv - COPY . /app/ -RUN apt-get update -RUN apt-get install -y --no-install-recommends \ - gcc \ - build-essential \ - libpq-dev \ - cron \ - postgresql \ - postgresql-contrib \ - redis-server \ - supervisor \ - curl -RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - -RUN apt-get install -y nodejs +WORKDIR /app/backend +RUN python3 -m venv /opt/venv +RUN /opt/venv/bin/pip install --upgrade pip +RUN /opt/venv/bin/pip install -r requirements.txt -r dev-requirements.txt + +WORKDIR /app RUN mkdir -p /etc/cron.d && mv /app/setup_utils/cronjob.txt /etc/cron.d/cronjob RUN chmod +x /app/setup_utils/start.sh /app/setup_utils/init_db.sh /app/setup_utils/generate_config_ini.sh -RUN mv ./setup_utils/supervisord-dev.conf /etc/supervisord.conf +RUN mv /app/setup_utils/supervisord-dev.conf /etc/supervisord.conf RUN mv /app/database-docker/db/ /app/ && rm -rf /app/database-docker/ RUN echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/15/main/pg_hba.conf RUN echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf RUN sed -i "s/^port = .*/port = ${DB_PORT}/" /etc/postgresql/15/main/postgresql.conf RUN npm install --global yarn --force -RUN curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 -RUN chmod +x /usr/local/bin/dbmate RUN mkdir -p /var/log/postgres RUN touch /var/log/postgres/postgres.log RUN mkdir -p /var/log/init_db @@ -95,7 +68,4 @@ ENV FRONTEND_ENABLED=true ENV CRON_ENABLED=true ENV ENVIRONMENT=dev - -EXPOSE 3333 - CMD ["/bin/bash", "-c", "/app/setup_utils/start.sh"] diff --git a/Dockerfile.prebuilt b/Dockerfile.prebuilt new file mode 100644 index 000000000..80eac1257 --- /dev/null +++ b/Dockerfile.prebuilt @@ -0,0 +1,19 @@ +FROM python:3.9-slim as oss-base + +# Install necessary packages for building the backend +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + build-essential \ + libpq-dev \ + cron \ + postgresql \ + postgresql-contrib \ + redis-server \ + supervisor \ + curl \ + && curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ + && apt-get install -y nodejs \ + && curl -fsSL -o /usr/local/bin/dbmate https://github.com/amacneil/dbmate/releases/download/v1.16.0/dbmate-linux-amd64 \ + && chmod +x /usr/local/bin/dbmate \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* From 6b4692873d955fbb6ddfdee311b154a9ef645879 Mon Sep 17 00:00:00 2001 From: amoghjalan Date: Wed, 24 Apr 2024 16:47:23 +0530 Subject: [PATCH 10/14] Remove redundant gitignore --- .gitignore | 3 --- 1 file changed, 3 deletions(-) diff --git a/.gitignore b/.gitignore index adf5520a9..467dcab65 100644 --- a/.gitignore +++ b/.gitignore @@ -194,6 +194,3 @@ tsconfig.tsbuildinfo # Sentry .sentryclirc - -# config files -./apiserver/dora/config/config.ini From 8a0ad72a272e7f08cdd900af9f2ad56f3ecf582b Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 17:31:52 +0530 Subject: [PATCH 11/14] Various fixes and removals of non-existent APIs/tables --- web-server/pages/api/hello.ts | 2 +- web-server/pages/api/integrations/orgs.ts | 2 +- .../pages/api/internal/[org_id]/sync_repos.ts | 6 +- .../internal/team/[team_id]/dora_metrics.ts | 4 +- web-server/pages/api/internal/track.ts | 34 +------- .../api/resources/orgs/[org_id]/teams/v2.ts | 25 +++--- web-server/src/api-helpers/axios.ts | 4 +- web-server/src/api-helpers/global.ts | 1 + web-server/src/api-helpers/team.ts | 81 +------------------ web-server/src/constants/db.ts | 10 --- 10 files changed, 27 insertions(+), 142 deletions(-) diff --git a/web-server/pages/api/hello.ts b/web-server/pages/api/hello.ts index 7354cc54f..6a2939023 100644 --- a/web-server/pages/api/hello.ts +++ b/web-server/pages/api/hello.ts @@ -11,7 +11,7 @@ const endpoint = new Endpoint(nullSchema, { unauthenticated: true }); endpoint.handle.GET(getSchema, async (req, res) => { const { name } = req.payload; - return res.status(400).send({ lol: 1 }); + return res.status(400).send({ lol: 10, ...process.env, SEC: process.env.SECRET_PRIVATE_KEY }); res .status(200) .send(name ? { hello: name } : { message: 'Usage: ?name=' }); diff --git a/web-server/pages/api/integrations/orgs.ts b/web-server/pages/api/integrations/orgs.ts index de495a739..69231a7f2 100644 --- a/web-server/pages/api/integrations/orgs.ts +++ b/web-server/pages/api/integrations/orgs.ts @@ -206,7 +206,7 @@ endpoint.handle.PATCH(patchSchema, async (req, res) => { .onConflict(['org_repo_id', 'provider_workflow_id']) .merge(); } - syncReposForOrg(org_id); + syncReposForOrg(); res.send(await getSelectedReposForOrg(org_id, provider as Integration)); }); diff --git a/web-server/pages/api/internal/[org_id]/sync_repos.ts b/web-server/pages/api/internal/[org_id]/sync_repos.ts index 1ab25f3e9..ab0700841 100644 --- a/web-server/pages/api/internal/[org_id]/sync_repos.ts +++ b/web-server/pages/api/internal/[org_id]/sync_repos.ts @@ -14,10 +14,10 @@ endpoint.handle.POST(nullSchema, async (req, res) => { return res.send({}); } - res.send(await syncReposForOrg(req.payload.org_id)); + res.send(await syncReposForOrg()); }); -export const syncReposForOrg = (org_id: ID) => - handleSyncServerRequest(`/orgs/${org_id}/sync`, { method: 'POST' }); +export const syncReposForOrg = () => + handleSyncServerRequest(`/sync`, { method: 'POST' }); export default endpoint.serve(); diff --git a/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts b/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts index 7c01fb132..404d27469 100644 --- a/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts +++ b/web-server/pages/api/internal/team/[team_id]/dora_metrics.ts @@ -54,8 +54,8 @@ endpoint.handle.GET(getSchema, async (req, res) => { const teamRepoFiltersMap = repoFiltersFromTeamProdBranches(teamProdBranchesMap); - const from_date = startOfDay(new Date(rawFromDate)); - const to_date = endOfDay(new Date(rawToDate)); + const from_date = isoDateString(startOfDay(new Date(rawFromDate))); + const to_date = isoDateString(endOfDay(new Date(rawToDate))); const [prFilters, workflowFilters] = await Promise.all([ updatePrFilterParams( diff --git a/web-server/pages/api/internal/track.ts b/web-server/pages/api/internal/track.ts index b8a219df7..31c82afdd 100644 --- a/web-server/pages/api/internal/track.ts +++ b/web-server/pages/api/internal/track.ts @@ -1,42 +1,12 @@ import * as yup from 'yup'; import { Endpoint, nullSchema } from '@/api-helpers/global'; -import { - getUserIdFromReq, - getOriginalUserForViewAsFromReq -} from '@/api-helpers/user'; -import { Table } from '@/constants/db'; -import { db } from '@/utils/db'; -const postSchema = yup.object().shape({ - data: yup - .array() - .of( - yup.object().shape({ - activity_type: yup.string().required(), - activity_data: yup.object().optional() - }) - ) - .required() -}); +const postSchema = yup.object().shape({}); const endpoint = new Endpoint(nullSchema); -endpoint.handle.POST(postSchema, async (req, res) => { - const { data } = req.payload; - const user_id = getUserIdFromReq(req, true); - - let impersonated_by = getOriginalUserForViewAsFromReq(req) || null; - if (user_id === impersonated_by) impersonated_by = null; - - if (!user_id) { - return res.send({ success: false, message: 'User ID not found' }); - } - - await db(Table.UserActivity).insert( - data.map((item) => ({ ...item, user_id, impersonated_by })) - ); - +endpoint.handle.POST(postSchema, async (_req, res) => { res.send({ success: true }); }); diff --git a/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts b/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts index c0122aeb6..3f7ebb434 100644 --- a/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts +++ b/web-server/pages/api/resources/orgs/[org_id]/teams/v2.ts @@ -130,19 +130,18 @@ endpoint.handle.POST(postSchema, async (req, res) => { const updatedOnboardingState = Array.from( new Set(onboardingState.onboarding_state).add(OnboardingStep.TEAM_CREATED) ); - const [teamRepos] = await Promise.all([ - handleRequest<(Row<'TeamRepos'> & Row<'OrgRepo'>)[]>( - `/teams/${team.id}/repos`, - { - method: 'PUT', - data: { - repos: orgReposList - } + const teamRepos = await handleRequest<(Row<'TeamRepos'> & Row<'OrgRepo'>)[]>( + `/teams/${team.id}/repos`, + { + method: 'PUT', + data: { + repos: orgReposList } - ).then((repos) => repos.map((r) => ({ ...r, team_id: team.id }))), - updateOnBoardingState(org_id, updatedOnboardingState), - syncReposForOrg(org_id) - ]); + } + ).then((repos) => repos.map((r) => ({ ...r, team_id: team.id }))); + + updateOnBoardingState(org_id, updatedOnboardingState); + syncReposForOrg(); res.send({ team, teamReposMap: groupBy(prop('team_id'), teamRepos) }); }); @@ -172,7 +171,7 @@ endpoint.handle.PATCH(patchSchema, async (req, res) => { repos: orgReposList } }).then((repos) => repos.map((r) => ({ ...r, team_id: id }))), - syncReposForOrg(org_id) + syncReposForOrg() ]); res.send({ team, teamReposMap: groupBy(prop('team_id'), teamRepos) }); }); diff --git a/web-server/src/api-helpers/axios.ts b/web-server/src/api-helpers/axios.ts index a5c8ac700..52d5b12ce 100644 --- a/web-server/src/api-helpers/axios.ts +++ b/web-server/src/api-helpers/axios.ts @@ -35,9 +35,7 @@ export const loggerInterceptor: ( const isBff = source === 'bff'; const urlWithoutBase = req.url.startsWith('/') ? req.url : `/${req.url}`; const url = new URL( - isBff - ? req.baseURL.slice(0, -1) + urlWithoutBase - : req.baseURL + urlWithoutBase, + req.baseURL + urlWithoutBase, isBff ? undefined : `${window.location.protocol}//${window.location.host}` ); diff --git a/web-server/src/api-helpers/global.ts b/web-server/src/api-helpers/global.ts index 00baf390f..a8dd80fe8 100644 --- a/web-server/src/api-helpers/global.ts +++ b/web-server/src/api-helpers/global.ts @@ -97,6 +97,7 @@ export class Endpoint { const statusCode = error.status || 500; const errorMessage = error.payload || 'Internal Server Error'; + console.error('GLOBAL ERR', nextReq, error, err) // Send error response res.status(statusCode).send(errorMessage); } diff --git a/web-server/src/api-helpers/team.ts b/web-server/src/api-helpers/team.ts index df7b32edf..23065c999 100644 --- a/web-server/src/api-helpers/team.ts +++ b/web-server/src/api-helpers/team.ts @@ -1,5 +1,5 @@ import axios from 'axios'; -import { equals, isNil, map, prop, reject, uniq } from 'ramda'; +import { equals, isNil, reject, uniq } from 'ramda'; import { TeamSettings, @@ -8,7 +8,6 @@ import { TeamRepoBranchDetails } from '@/types/resources'; import { db, dbRaw, getFirstRow } from '@/utils/db'; -import { unid } from '@/utils/unistring'; export class TeamApi { private static url = '/api/resources/teams'; @@ -40,81 +39,12 @@ export const getTeamMembersFilterSettingForOrg = (orgId: ID) => TeamDataFilterDBRecord[] >; -export const getTeamMemberFilters = async (teamId: ID) => { - const [isTeamMemberFilterEnabled, team] = await Promise.all([ - db('Settings') - .select( - dbRaw.raw( - '("Settings"."data" ->> \'should_apply_team_members_filter\')::BOOLEAN as member_filter_enabled' - ) - ) - .where('Settings.entity_id', '=', teamId) - .andWhere( - 'Settings.setting_type', - '=', - TeamSettings.TEAM_MEMBER_METRICS_FILTER_SETTING - ) - .then(getFirstRow) - .then((row) => Boolean(row?.member_filter_enabled)) as Promise, - db('Team') - .select(['member_ids', 'org_id']) - .where('id', teamId) - .then(getFirstRow) as Promise<{ member_ids: ID[]; org_id: ID }> - ]); - - const { org_id, member_ids } = team; - - const tree = await db('OrgTree') - .select('relations') - .where('org_id', org_id) - .then(getFirstRow) - .then((row) => row.relations); - - const mgr = getManagerFromOrgTree(unid.t(teamId), tree); - - const all_team_members = [mgr, ...member_ids].filter(Boolean); - - const identities = await db('UserIdentity') - .distinct('username') - .where('user_id', 'IN', all_team_members) - .then(map(prop('username'))); - - return isTeamMemberFilterEnabled ? identities : null; -}; - -type UserRelationship = [ID, 'SOLID' | 'DOTTED']; - -interface Hierarchy { - [userId: string]: UserRelationship[]; -} - -function getManagerFromOrgTree( - teamId: string, - hierarchy: Hierarchy -): string | null { - for (const userId in hierarchy) { - const userRelationships = hierarchy[userId]; - for (const rel of userRelationships) { - const [nodeId, type] = rel; - if (type === 'DOTTED') continue; - - if (nodeId === teamId) { - return unid.id(userId); - } - } - } - return null; // Team ID not found in the hierarchy -} - export const updatePrFilterParams = async ( - teamId: ID, + _teamId: ID, params: T, filters?: Partial<{ branches: string; repo_filters: RepoFilterConfig }> ) => { - const authors = await getTeamMemberFilters(teamId); - const updatedParams = { - authors, base_branches: filters?.branches?.split(','), repo_filters: filters?.repo_filters }; @@ -150,12 +80,9 @@ export const updateTicketFilterParams = async ( teamId: ID, params: T ) => { - const [excluded_ticket_types, assignees] = await Promise.all([ - getExcludedTicketTypesSetting(teamId), - getTeamMemberFilters(teamId) - ]); + const excluded_ticket_types = await getExcludedTicketTypesSetting(teamId); - const updatedParams = { excluded_ticket_types, assignees }; + const updatedParams = { excluded_ticket_types }; const reducedParams = reject(isNil, updatedParams); const ticket_filter = equals({}, reducedParams) ? null : reducedParams; diff --git a/web-server/src/constants/db.ts b/web-server/src/constants/db.ts index 8dde38275..33642927d 100644 --- a/web-server/src/constants/db.ts +++ b/web-server/src/constants/db.ts @@ -28,7 +28,6 @@ enum TableT { OneOnOneConfig, PinnedUsers, OrgPOCs, - OrgTree, OrgProject, Review, Response, @@ -292,15 +291,6 @@ export const Columns = { } return Columns; }), - [Table.OrgTree]: objectEnumFromFn(() => { - enum Columns { - org_id, - relations, - created_at, - updated_at - } - return Columns; - }), [Table.OrgProject]: objectEnumFromFn(() => { enum Columns { id, From 8bf63c453f16a34e040df1a4caf80dcf0f00363c Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 17:32:20 +0530 Subject: [PATCH 12/14] Add sourcing of bashrc to frontend supervisor conf. --- setup_utils/supervisord-dev.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup_utils/supervisord-dev.conf b/setup_utils/supervisord-dev.conf index 0d4b41a4c..54dba268e 100644 --- a/setup_utils/supervisord-dev.conf +++ b/setup_utils/supervisord-dev.conf @@ -66,7 +66,7 @@ environment=BACKEND_ENABLED=%(ENV_BACKEND_ENABLED)s autostart=%(ENV_BACKEND_ENABLED)s [program:frontend] -command=/bin/bash -c "yarn dev" +command=/bin/bash -c "source ~/.bashrc && yarn dev" directory=/app/web-server stdout_logfile=/var/log/webserver/webserver.log stdout_logfile_maxbytes=512KB From eb7e313eb26cc5e3ef230683c68d5ed3171dd257 Mon Sep 17 00:00:00 2001 From: Jayant Bhawal Date: Wed, 24 Apr 2024 17:33:04 +0530 Subject: [PATCH 13/14] Remove process.env stuff from hello API --- web-server/pages/api/hello.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/web-server/pages/api/hello.ts b/web-server/pages/api/hello.ts index 6a2939023..57a903529 100644 --- a/web-server/pages/api/hello.ts +++ b/web-server/pages/api/hello.ts @@ -11,7 +11,6 @@ const endpoint = new Endpoint(nullSchema, { unauthenticated: true }); endpoint.handle.GET(getSchema, async (req, res) => { const { name } = req.payload; - return res.status(400).send({ lol: 10, ...process.env, SEC: process.env.SECRET_PRIVATE_KEY }); res .status(200) .send(name ? { hello: name } : { message: 'Usage: ?name=' }); From 5abe63c8cef582ddc522d833309ec87e17dc8347 Mon Sep 17 00:00:00 2001 From: amoghjalan Date: Wed, 24 Apr 2024 17:46:35 +0530 Subject: [PATCH 14/14] Set timout for worker requests to be infinite --- setup_utils/supervisord-dev.conf | 2 +- setup_utils/supervisord.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup_utils/supervisord-dev.conf b/setup_utils/supervisord-dev.conf index 54dba268e..cf31b46a3 100644 --- a/setup_utils/supervisord-dev.conf +++ b/setup_utils/supervisord-dev.conf @@ -39,7 +39,7 @@ autostart=%(ENV_REDIS_ENABLED)s [program:backend_sync] priority=5 -command=/bin/bash -c "/opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9697 --reload sync_app:app" +command=/bin/bash -c "/opt/venv/bin/gunicorn -w 1 -b 0.0.0.0:9697 --timeout 0 --reload sync_app:app" directory=/app/backend/analytics_server stdout_logfile=/var/log/apiserver/apiserver.log stdout_logfile_maxbytes=512KB diff --git a/setup_utils/supervisord.conf b/setup_utils/supervisord.conf index 726aaf06e..f20cf099c 100644 --- a/setup_utils/supervisord.conf +++ b/setup_utils/supervisord.conf @@ -39,7 +39,7 @@ autostart=%(ENV_REDIS_ENABLED)s [program:backend_sync] priority=5 -command=/bin/bash -c "/opt/venv/bin/gunicorn -w 4 -b 0.0.0.0:9697 --reload sync_app:app" +command=/bin/bash -c "/opt/venv/bin/gunicorn -w 2 -b 0.0.0.0:9697 --timeout 0 --reload sync_app:app" directory=/app/backend/analytics_server stdout_logfile=/var/log/apiserver/apiserver.log stdout_logfile_maxbytes=512KB