chore(repo-structure): Docker images and stacks organized and clarifi…

…ed (#9530)

.env.example

@@ -74,9 +74,9 @@ POSTGRES_USE_PGVECTOR=true
 # POSTGRES_SSL_DIR='/var/lib/postgresql'
 REDIS_PASSWORD=''
 REDIS_URL='redis://localhost:6379'
-# REDIS_TLS_CERT_FILE=./certs/redis.crt
-# REDIS_TLS_KEY_FILE=./certs/redis.key
-# REDIS_TLS_CA_FILE=./certs/redisCA.crt
+# REDIS_TLS_CERT_FILE=./docker/stacks/development/redis/certs/redis.crt
+# REDIS_TLS_KEY_FILE=./docker/stacks/development/redis/certs/redis.key
+# REDIS_TLS_CA_FILE=./docker/stacks/development/redis/certs/redisCA.crt
 # REDIS_TLS_REJECT_UNAUTHORIZED='false'
 RETHINKDB_URL='rethinkdb://localhost:28015/actionDevelopment'
 RETHINKDB_SSL='false'
@@ -119,9 +119,9 @@ RETHINKDB_SSL='false'
 # RECALL_AI_KEY=''
 # SLACK_CLIENT_ID='key_SLACK_CLIENT_ID'
 # SLACK_CLIENT_SECRET='key_SLACK_CLIENT_SECRET'
-# STRIPE_SECRET_KEY='sk_test_4eC39HqLyjWDarjtT1zdp7dc'
-# STRIPE_PUBLISHABLE_KEY='pk_test_TYooMQauvdEDq54NiTphI7jx'
-# STRIPE_WEBHOOK_SECRET='sk_test_4eC39HqLyjWDarjtT1zdp7dc'
+# STRIPE_SECRET_KEY=''
+# STRIPE_PUBLISHABLE_KEY=''
+# STRIPE_WEBHOOK_SECRET=''
 
 # MAIL
 # MAIL GLOBALS. PROVIDER: mailgun | google | debug | smtp

.github/workflows/README.md

@@ -2,7 +2,7 @@
 
 ## Runners
 
-To run `docker-build.yml`, GitHub requires using a larger runner.
+To run `build.yml`, GitHub requires using a larger runner.
 This is because we're webpackifying all the code in node_modules into a single `.js.`.
 Doing all that transpiling can take a LOT of memory. 8GB+.
 At this time, large GitHub-hosted runners are in beta.

.github/workflows/build.yml

@@ -12,8 +12,8 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
   cancel-in-progress: true
 env:
-  PARABOL_DOCKERFILE: ./docker/parabol-ubi/docker-build/dockerfiles/basic.dockerfile
-  PARABOL_BUILD_ENV_PATH: docker/parabol-ubi/docker-build/environments/pipeline
+  PARABOL_DOCKERFILE: ./docker/images/parabol-ubi/dockerfiles/basic.dockerfile
+  PARABOL_BUILD_ENV_PATH: docker/images/parabol-ubi/environments/pipeline
 jobs:
   build:
     runs-on: ubuntu-8cores
@@ -22,8 +22,7 @@ jobs:
       id-token: "write"
     services:
       postgres:
-        # Image is pinned to v15, OK since it's just for testing
-        image: ankane/pgvector
+        image: pgvector/pgvector:pg15
         # This env variables must be the same in the file PARABOL_BUILD_ENV_PATH
         env:
           POSTGRES_PASSWORD: "temppassword"
@@ -106,7 +105,7 @@ jobs:
           username: "oauth2accesstoken"
           password: "${{ steps.auth.outputs.access_token }}"
       - name: Push build to dev
-        uses: docker/build-push-action@v4
+        uses: docker/images-push-action@v4
         with:
           network: host
           allow: network.host

.github/workflows/test.yml

@@ -8,7 +8,7 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 env:
-  PARABOL_BUILD_ENV_PATH: docker/parabol-ubi/docker-build/environments/pipeline
+  PARABOL_BUILD_ENV_PATH: docker/images/parabol-ubi/environments/pipeline
 jobs:
   test:
     runs-on: ubuntu-8cores

docker/parabol-ubi/docker-build/README.md → docker/images/parabol-ubi/README.md

@@ -21,9 +21,9 @@ Recommended:
 | `postgresql_tag`     | PostgreSQL version from the [Docker image](https://hub.docker.com/_/postgres)                                           | `Any tag`                                             | `15.4`                                                              |
 | `rethinkdb_tag`      | RethinkDB version from the [Docker image](https://hub.docker.com/_/rethinkdb)                                           | `Any tag`                                             | `2.4.2`                                                             |
 | `redis_tag`          | Redis version from the [Docker image](https://hub.docker.com/_/redis)                                                   | `Any tag`                                             | `7.0-alpine`                                                        |
-| `_BUILD_ENV_PATH`    | File `.env` used by the application during the build process                                                            | `Relative path from the root level of the repository` | `docker/parabol-ubi/docker-build/environments/basic-env`             |
+| `_BUILD_ENV_PATH`    | File `.env` used by the application during the build process                                                            | `Relative path from the root level of the repository` | `docker/parabol-ubi/environments/basic-env`             |
 | `_NODE_VERSION`      | Node version, used by Docker to use the Docker image node:\_NODE_VERSION as base image to build                         | `Same as in root package.json`                        |                                                                     |
-| `_DOCKERFILE`        | Dockerfile used to build the image                                                                                      | `Relative path from the root level of the repository` | `./docker/parabol-ubi/docker-build/dockerfiles/basic.dockerfile` |
+| `_DOCKERFILE`        | Dockerfile used to build the image                                                                                      | `Relative path from the root level of the repository` | `./docker/parabol-ubi/dockerfiles/basic.dockerfile` |
 | `_DOCKER_REPOSITORY` | The destination repository                                                                                              | `String`                                              | `parabol`                                                           |
 | `_DOCKER_TAG`        | Tag for the produced image                                                                                              | `String`                                              |                                                                     |
 
@@ -33,9 +33,9 @@ Example of variables:
 export postgresql_tag=15.4; \
 export rethinkdb_tag=2.4.2; \
 export redis_tag=7.0-alpine; \
-export _BUILD_ENV_PATH=docker/parabol-ubi/docker-build/environments/basic-env; \
+export _BUILD_ENV_PATH=docker/parabol-ubi/environments/basic-env; \
 export _NODE_VERSION=$(jq -r -j '.engines.node|ltrimstr("^")' package.json); \
-export _DOCKERFILE=./docker/parabol-ubi/docker-build/dockerfiles/basic.dockerfile; \
+export _DOCKERFILE=./docker/parabol-ubi/dockerfiles/basic.dockerfile; \
 export _DOCKER_REPOSITORY=parabol; \
 export _DOCKER_TAG=test-image
 ```

docker/parabol-ubi/docker-build/dockerfiles/basic.dockerfile → docker/images/parabol-ubi/dockerfiles/basic.dockerfile

@@ -7,8 +7,8 @@ ENV HOME=/home/node \
 ENV NPM_CONFIG_PREFIX=/home/node/.npm-global
 ENV PORT=3000
 
-COPY --chown=node --chmod=755 docker/parabol-ubi/docker-build/entrypoints/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
-COPY --chown=node docker/parabol-ubi/docker-build/tools/ip-to-server_id ${HOME}/tools/ip-to-server_id
+COPY --chown=node --chmod=755 docker/parabol-ubi/entrypoints/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
+COPY --chown=node docker/parabol-ubi/tools/ip-to-server_id ${HOME}/tools/ip-to-server_id
 
 # Required for pushToCDN to work with FILE_STORE_PROVIDER set to 'local'
 RUN mkdir -p ${HOME}/parabol/self-hosted && \

docker/parabol-ubi/docker-build/environments/basic-env → docker/images/parabol-ubi/environments/basic-env

@@ -4,7 +4,7 @@ NODE_ENV='production'
 NODE_EXTRA_CA_CERTS=''
 PROTO='https'
 PORT='3000'
-# Database configurations must be the same used in the docker-build.yml Github workflow
+# Database configurations must be the same used in the build.yml Github workflow
 POSTGRES_PASSWORD='temppassword'
 POSTGRES_USER='tempuser'
 POSTGRES_DB='tempdb'

docker/parabol-ubi/docker-build/environments/pipeline → docker/images/parabol-ubi/environments/pipeline

@@ -36,7 +36,7 @@ PGADMIN_DEFAULT_EMAIL=''
 PGADMIN_DEFAULT_PASSWORD=''
 PGSSLMODE=''
 PORT='3000'
-# Database configurations must be the same used in the docker-build.yml Github workflow
+# Database configurations must be the same used in the build.yml Github workflow
 POSTGRES_PASSWORD='temppassword'
 POSTGRES_USER='tempuser'
 POSTGRES_DB='tempdb'

packages/server/postgres/Dockerfile → docker/images/postgres/Dockerfile

@@ -1,15 +1,17 @@
 FROM postgres:15.4
+ARG PGVECTOR_VERSION=v0.6.1
+ARG PSQL_MAJOR_VERSION=15
 
 ADD extensions /extensions
 
 RUN apt-get update && apt-get install -y \
     build-essential \
     locales \
-    postgresql-server-dev-15 \
+    postgresql-server-dev-${PSQL_MAJOR_VERSION} \
     git
 
-RUN cd /extensions/postgres-json-schema && make install && make installcheck
-RUN git clone --branch v0.5.0 \
+# PGVector
+RUN git clone --branch ${PGVECTOR_VERSION} \
        https://github.com/pgvector/pgvector.git /extensions/pgvector && \
        cd extensions/pgvector && make clean && make && make install
 

docker/images/postgres/extensions/install.sql

@@ -0,0 +1 @@
+CREATE EXTENSION IF NOT EXISTS "vector";

docker/stacks/development/README.md

@@ -0,0 +1,43 @@
+# Development stack
+
+> ⚠️ **Parabol does not provide any support on this stack**. Use it under your own resposibility.
+
+## General notes
+
+- **This stack is not meant for production use.** It is our development stack and can change at any moment, have errors and incorporate and remove components we are testing without any notice.
+- This stack is designed to be managed using `yarn db:start` and `yarn db:stop` to start the databases. The application can use it, starting with either `yarn dev` or building the application and using `yarn start`.
+
+## Components
+
+- **Datadog agent:** additional configuration can be added in the folder `datadog/dd-conf.d`.
+- **RethinkDB:** ports 28015 and 8080 (console) exposed to communicate with the cluster. Data mounted in a volume rethinkdb-data.
+- **Postgres:** container built from a Dockerfile in [docker/images/postgres](docker/images/postgres), that incorporates some extra extensions used by the application. Exposed through port 5432 and with the data mounted in a volume postgres-data.
+- **PGAdmin:** available on 5050 with credentials on the `.env` file. Connect using the values of `PGADMIN_DEFAULT_EMAIL` and `PGADMIN_DEFAULT_PASSWORD` from the `.env`. Data mounted on a volume pgadmin-data.
+- **Redis:** available on 6379 with the data mounted on a volume redis-data.
+- **Redis Commander:** available on 8081.
+- **Text Embedding Inference:** toolkit to deploy and serve open source text embeddings and sequence classification models. Exposed on 3040. More information in [their Github](https://github.com/huggingface/text-embeddings-inference).
+
+### Configue PGAdmin
+
+- pgadmin is at [http://localhost:5050](http://localhost:5050)
+- Connect using the values of `PGADMIN_DEFAULT_EMAIL` and `PGADMIN_DEFAULT_PASSWORD` from your `.env`
+- Click **Add New Server** and fill out the forms with your `.env` values
+
+  - General.name = POSTGRES_DB
+  - Connection.host = 'postgres' (hardcoded from docker-compose dev.yml, not from .env!)
+  - Connection.username = POSTGRES_USER
+  - Connection.password = POSTGRES_PASSWORD
+  - Connection.maintenanceDatabase = POSTGRES_DB
+  - Connection.port = POSTGRES_PORT
+
+### Postgres
+
+#### Too many connections
+
+Sometimes pg pool will hit its connection limit. This should never happen in prod, but happens on occassion in dev.
+You'll know it's happening because PG will say there are too many connections.
+To fix, you can run the following SQL to remove all the connections except the one that is running the script
+
+```sql
+select pg_terminate_backend(pid) from pg_stat_activity where datname='parabol-saas' AND pid <> pg_backend_pid();
+```

docker/dev.yml → docker/stacks/development/docker-compose.yml

@@ -1,10 +1,11 @@
-version: "3.7"
+version: "3.9"
+name: parabol-dev
 
 services:
   datadog:
     image: gcr.io/datadoghq/agent:7
     restart: unless-stopped
-    env_file: ../.env
+    env_file: ../../../.env
     ports:
       - "8126:8126"
     networks:
@@ -13,7 +14,9 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - /proc/:/host/proc/:ro
       - /sys/fs/cgroup:/host/sys/fs/cgroup:ro
-  db:
+      - ".datadog/dd-conf.d:/etc/datadog-agent/conf.d/local.d/"
+      - "../../../dev/logs:/var/log/datadog/logs"
+  rethinkdb:
     image: rethinkdb:2.4.2
     restart: unless-stopped
     ports:
@@ -26,23 +29,20 @@ services:
       - parabol-network
   postgres:
     build:
-      context: "../packages/server/postgres"
+      context: "../../images/postgres"
     restart: unless-stopped
-    env_file: ../.env
+    env_file: ../../../.env
     ports:
       - "5432:5432"
     volumes:
-      - "../packages/server/postgres/postgres.conf:/usr/local/etc/postgres/postgres.conf"
       - "postgres-data:/var/lib/postgresql/data"
-    command: "postgres -c config_file=/usr/local/etc/postgres/postgres.conf"
     networks:
       - parabol-network
   pgadmin:
-    container_name: pgadmin_container
-    image: dpage/pgadmin4:latest
+    image: dpage/pgadmin4:8.3
     depends_on:
       - postgres
-    env_file: ../.env
+    env_file: ../../../.env
     volumes:
       - "pgadmin-data:/var/lib/pgadmin"
     ports:
@@ -60,18 +60,16 @@ services:
     networks:
       - parabol-network
   redis-commander:
-    container_name: redis_commander
-    image: ghcr.io/joeferner/redis-commander:latest
+    image: ghcr.io/joeferner/redis-commander:0.8.1
     hostname: redis-commander
     restart: unless-stopped
     environment:
       - REDIS_HOSTS=local:redis:6379
     ports:
-      - "8082:8081"
+      - "8081:8081"
     networks:
       parabol-network:
   text-embeddings-inference:
-    container_name: text-embeddings-inference
     image: ghcr.io/huggingface/text-embeddings-inference:cpu-0.6
     command:
       - "--model-id=llmrails/ember-v1"