From f4fb0ff6a9ff9add6a44d43e9adc34a9c02d2583 Mon Sep 17 00:00:00 2001 From: Phoevos Kalemkeris Date: Wed, 15 Jan 2025 15:26:46 +0000 Subject: [PATCH 1/2] compose: Unset 'container_name' for all services Remove hardcoded container names from all services in the docker-compose files to allow multiple instances of the same service to run in parallel without conflicts. This is important to allow both scaling individual services as well as running multiple instances of the CMS stack locally for testing purposes. Signed-off-by: Phoevos Kalemkeris --- docker-compose-auth.yml | 1 - docker-compose-celery.yml | 3 --- docker-compose-dev.yml | 15 ++++++++++----- docker-compose-log.yml | 3 --- docker-compose-mlflow.yml | 9 +++------ docker-compose-mon.yml | 4 ---- docker-compose-proxy.yml | 1 - docker-compose.yml | 6 ------ tests/load/docker-compose.yml | 6 ++++-- 9 files changed, 17 insertions(+), 31 deletions(-) diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml index cb27dee..0fe0e87 100644 --- a/docker-compose-auth.yml +++ b/docker-compose-auth.yml @@ -4,7 +4,6 @@ services: auth-db: image: postgres:14 - container_name: cms_auth-db restart: always networks: - cogstack-model-serve_cms diff --git a/docker-compose-celery.yml b/docker-compose-celery.yml index bce8ef8..d803809 100644 --- a/docker-compose-celery.yml +++ b/docker-compose-celery.yml @@ -12,7 +12,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_worker networks: - cms volumes: @@ -32,7 +31,6 @@ services: build: context: ./ dockerfile: ./docker/celery/Dockerfile-Dashboard - container_name: cms_worker_dashboard networks: - cms ports: @@ -45,7 +43,6 @@ services: redis: image: redis:7 - container_name: cms_redis networks: - cms ports: diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 699e5ac..576da4d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -8,7 +8,8 @@ services: file: ./docker-compose.yml service: medcat-snomed image: cogstack-model-serve_medcat-snomed:dev - container_name: dev-cms_medcat-snomed + labels: + - org.cogstack.model-serve.env=dev volumes: - ${MODEL_PACKAGE_FULL_PATH}:/app/model/model.zip:ro environment: @@ -37,7 +38,8 @@ services: file: ./docker-compose.yml service: medcat-icd10 image: cogstack-model-serve_medcat-icd10:dev - container_name: dev-cms_medcat-icd10 + labels: + - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= @@ -64,7 +66,8 @@ services: file: ./docker-compose.yml service: de-identification image: cogstack-model-serve_de-identification:dev - container_name: dev-cms_trf-deid + labels: + - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= @@ -91,7 +94,8 @@ services: file: ./docker-compose.yml service: medcat-deid image: cogstack-model-serve_medcat-deid:dev - container_name: dev-cms_medcat-deid + labels: + - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= @@ -118,7 +122,8 @@ services: file: ./docker-compose.yml service: medcat-umls image: cogstack-model-serve_medcat-umls:dev - container_name: dev-cms_medcat-umls + labels: + - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= diff --git a/docker-compose-log.yml b/docker-compose-log.yml index a12c198..e757232 100644 --- a/docker-compose-log.yml +++ b/docker-compose-log.yml @@ -4,7 +4,6 @@ services: mongodb: image: mongo:5.0 - container_name: cms_mongodb volumes: - mongodb_data:/data/db restart: always @@ -20,7 +19,6 @@ services: opensearch: image: opensearchproject/opensearch:2.4.0 - container_name: cms_opensearch environment: - OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g - bootstrap.memory_lock=true @@ -47,7 +45,6 @@ services: graylog: image: graylog/graylog:5.0 - container_name: cms_graylog depends_on: opensearch: condition: "service_healthy" diff --git a/docker-compose-mlflow.yml b/docker-compose-mlflow.yml index af1121c..4fc713e 100644 --- a/docker-compose-mlflow.yml +++ b/docker-compose-mlflow.yml @@ -4,7 +4,6 @@ services: mlflow-db: image: postgres:14 - container_name: cms_mlflow-db restart: always networks: - cogstack-model-serve_cms @@ -24,7 +23,6 @@ services: minio: image: minio/minio:RELEASE.2022-06-30T20-58-09Z - container_name: cms_minio restart: always networks: - cogstack-model-serve_cms @@ -71,7 +69,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_mlflow-ui restart: always networks: - cogstack-model-serve_cms @@ -80,7 +77,7 @@ services: environment: - MLFLOW_DB_USERNAME=$MLFLOW_DB_USERNAME - MLFLOW_DB_PASSWORD=$MLFLOW_DB_PASSWORD - - MLFLOW_BACKEND_STORE_URI=postgresql://$MLFLOW_DB_USERNAME:$MLFLOW_DB_PASSWORD@cms_mlflow-db:5432/mlflow-backend-store + - MLFLOW_BACKEND_STORE_URI=postgresql://$MLFLOW_DB_USERNAME:$MLFLOW_DB_PASSWORD@mlflow-db:5432/mlflow-backend-store - MLFLOW_TRACKING_URI=http://localhost:5000 - ARTIFACTS_DESTINATION=s3://cms-model-bucket - MLFLOW_S3_ENDPOINT_URL=http://minio:9000 @@ -121,7 +118,6 @@ services: - NO_PROXY=$NO_PROXY profiles: - wip - container_name: cms_mlflow-deployments restart: always networks: - cogstack-model-serve_cms @@ -153,7 +149,8 @@ services: - NO_PROXY=$NO_PROXY profiles: - wip - container_name: cms_mlflow-${MODEL_NAME:-model} + labels: + - org.cogstack.model-serve.model-name=${MODEL_NAME:-model} restart: always networks: - cogstack-model-serve_cms diff --git a/docker-compose-mon.yml b/docker-compose-mon.yml index 86dc8fd..4683f39 100644 --- a/docker-compose-mon.yml +++ b/docker-compose-mon.yml @@ -4,7 +4,6 @@ services: prometheus: image: prom/prometheus:v2.41.0 - container_name: cms_prometheus restart: always networks: - cogstack-model-serve_cms @@ -29,7 +28,6 @@ services: grafana: image: grafana/grafana:9.3.2-ubuntu - container_name: cms_grafana restart: always networks: - cogstack-model-serve_cms @@ -51,7 +49,6 @@ services: alertmanager: image: prom/alertmanager:v0.25.0 - container_name: cms_alertmanager restart: always networks: - cogstack-model-serve_cms @@ -73,7 +70,6 @@ services: cadvisor: image: gcr.io/cadvisor/cadvisor:v0.46.0 - container_name: cms_cadvisor restart: always networks: - cogstack-model-serve_cms diff --git a/docker-compose-proxy.yml b/docker-compose-proxy.yml index 4a9bf56..ea29378 100644 --- a/docker-compose-proxy.yml +++ b/docker-compose-proxy.yml @@ -10,7 +10,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_proxy restart: always networks: - cogstack-model-serve_cms diff --git a/docker-compose.yml b/docker-compose.yml index 4d17877..804efbe 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_medcat-snomed restart: always networks: - cms @@ -63,7 +62,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_medcat-icd10 restart: always networks: - cms @@ -113,7 +111,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_trf-deid restart: always networks: - cms @@ -162,7 +159,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_medcat-deid restart: always networks: - cms @@ -212,7 +208,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_medcat-umls restart: always networks: - cms @@ -262,7 +257,6 @@ services: - HTTP_PROXY=$HTTP_PROXY - HTTPS_PROXY=$HTTPS_PROXY - NO_PROXY=$NO_PROXY - container_name: cms_huggingface-ner restart: always networks: - cms diff --git a/tests/load/docker-compose.yml b/tests/load/docker-compose.yml index 9198471..97d6ffa 100644 --- a/tests/load/docker-compose.yml +++ b/tests/load/docker-compose.yml @@ -3,7 +3,8 @@ version: "3" services: master: image: locustio/locust:2.26.0 - container_name: cms_load_master + labels: + - org.cogstack.model-serve.locust=master networks: - cogstack-model-serve_cms ports: @@ -25,7 +26,8 @@ services: worker: image: locustio/locust:2.26.0 - container_name: cms_load_worker + labels: + - org.cogstack.model-serve.locust=worker networks: - cogstack-model-serve_cms environment: From a5d74e14ef4598723387e18f1f85a901092c2505 Mon Sep 17 00:00:00 2001 From: Phoevos Kalemkeris Date: Thu, 16 Jan 2025 12:51:20 +0000 Subject: [PATCH 2/2] compose: Set top-level project name Set the top-level project name in the docker-compose files so that it can be used as the default prefix for container names. Use 'cms' for all stacks apart from the load tests (i.e. 'cms-load') and the dev deployment (i.e. 'dev-cms'). The project name can be overridden on deployment through the '-p' command line flag or by setting the 'COMPOSE_PROJECT_NAME' environment variable. Signed-off-by: Phoevos Kalemkeris --- docker-compose-auth.yml | 2 ++ docker-compose-celery.yml | 2 ++ docker-compose-dev.yml | 12 ++---------- docker-compose-log.yml | 2 ++ docker-compose-mlflow.yml | 2 ++ docker-compose-mon.yml | 2 ++ docker-compose-proxy.yml | 2 ++ docker-compose.yml | 2 ++ tests/load/docker-compose.yml | 2 ++ 9 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docker-compose-auth.yml b/docker-compose-auth.yml index 0fe0e87..56a45a5 100644 --- a/docker-compose-auth.yml +++ b/docker-compose-auth.yml @@ -1,5 +1,7 @@ version: "3.6" +name: cms + services: auth-db: diff --git a/docker-compose-celery.yml b/docker-compose-celery.yml index d803809..6d94954 100644 --- a/docker-compose-celery.yml +++ b/docker-compose-celery.yml @@ -1,5 +1,7 @@ version: "3.6" +name: cms + services: worker: build: diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 576da4d..11746ab 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,6 +1,8 @@ # This is for spinning up core services as single hosts in the DEV environment version: "3.6" +name: dev-cms + services: medcat-snomed: @@ -8,8 +10,6 @@ services: file: ./docker-compose.yml service: medcat-snomed image: cogstack-model-serve_medcat-snomed:dev - labels: - - org.cogstack.model-serve.env=dev volumes: - ${MODEL_PACKAGE_FULL_PATH}:/app/model/model.zip:ro environment: @@ -38,8 +38,6 @@ services: file: ./docker-compose.yml service: medcat-icd10 image: cogstack-model-serve_medcat-icd10:dev - labels: - - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= @@ -66,8 +64,6 @@ services: file: ./docker-compose.yml service: de-identification image: cogstack-model-serve_de-identification:dev - labels: - - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= @@ -94,8 +90,6 @@ services: file: ./docker-compose.yml service: medcat-deid image: cogstack-model-serve_medcat-deid:dev - labels: - - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= @@ -122,8 +116,6 @@ services: file: ./docker-compose.yml service: medcat-umls image: cogstack-model-serve_medcat-umls:dev - labels: - - org.cogstack.model-serve.env=dev environment: - BASE_MODEL_FULL_PATH=$MODEL_PACKAGE_FULL_PATH - AWS_ACCESS_KEY_ID= diff --git a/docker-compose-log.yml b/docker-compose-log.yml index e757232..19dfa76 100644 --- a/docker-compose-log.yml +++ b/docker-compose-log.yml @@ -1,5 +1,7 @@ version: "3.6" +name: cms + services: mongodb: diff --git a/docker-compose-mlflow.yml b/docker-compose-mlflow.yml index 4fc713e..7c5f3ae 100644 --- a/docker-compose-mlflow.yml +++ b/docker-compose-mlflow.yml @@ -1,5 +1,7 @@ version: "3.6" +name: cms + services: mlflow-db: diff --git a/docker-compose-mon.yml b/docker-compose-mon.yml index 4683f39..c76d0e5 100644 --- a/docker-compose-mon.yml +++ b/docker-compose-mon.yml @@ -1,5 +1,7 @@ version: "3.6" +name: cms + services: prometheus: diff --git a/docker-compose-proxy.yml b/docker-compose-proxy.yml index ea29378..4a6becd 100644 --- a/docker-compose-proxy.yml +++ b/docker-compose-proxy.yml @@ -1,5 +1,7 @@ version: "3.6" +name: cms + services: proxy: diff --git a/docker-compose.yml b/docker-compose.yml index 804efbe..39435ab 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,7 @@ version: "3.6" +name: cms + services: medcat-snomed: diff --git a/tests/load/docker-compose.yml b/tests/load/docker-compose.yml index 97d6ffa..0b31d02 100644 --- a/tests/load/docker-compose.yml +++ b/tests/load/docker-compose.yml @@ -1,5 +1,7 @@ version: "3" +name: cms-load + services: master: image: locustio/locust:2.26.0