From 06f880691b933f082cc4eb4edf730cc50455c5cd Mon Sep 17 00:00:00 2001 From: Nathan Levesque Date: Mon, 9 Sep 2024 17:56:32 -0400 Subject: [PATCH] Update docker-compose config to support OS cluster --- .devcontainer/devcontainer.json | 2 +- Dockerfile | 4 +- README.md | 12 ++ docker-compose.apps.yml | 68 ++++++++ docker-compose.codespaces.yml | 93 ++--------- docker-compose.opensearch.base.yml | 21 +++ docker-compose.opensearch.cluster.apps.yml | 17 ++ docker-compose.opensearch.cluster.yml | 48 ++++++ ...er-compose.opensearch.single-node.apps.yml | 9 + docker-compose.opensearch.single-node.yml | 17 ++ docker-compose.services.yml | 83 ++++++++++ docker-compose.yml | 155 ++---------------- env/backend.env | 2 +- env/ci.env | 2 +- env/codespaces.env | 2 +- 15 files changed, 304 insertions(+), 231 deletions(-) create mode 100644 docker-compose.apps.yml create mode 100644 docker-compose.opensearch.base.yml create mode 100644 docker-compose.opensearch.cluster.apps.yml create mode 100644 docker-compose.opensearch.cluster.yml create mode 100644 docker-compose.opensearch.single-node.apps.yml create mode 100644 docker-compose.opensearch.single-node.yml create mode 100644 docker-compose.services.yml diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 513c37e424..a26590ed9f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -4,7 +4,7 @@ "runServices": [ "watch", "web", - "opensearch-node-mitopen", + "opensearch-node-mitopen-1", "db", "tika", "celery", diff --git a/Dockerfile b/Dockerfile index 3a4753e25e..9d3cf6721e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,6 +56,6 @@ RUN apt-get clean && apt-get purge USER mitodl -EXPOSE 8063 -ENV PORT 8063 +EXPOSE 8061 +ENV PORT 8061 CMD uwsgi uwsgi.ini diff --git a/README.md b/README.md index 49f5212b47..4c2e992743 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,18 @@ To enable searching the course catalog on opensearch, run through these steps: 3. Once created and with `docker compose up` running, hit this endpoint in your browser to see if the index exists: `http://localhost:9101/discussions_local_all_default/_search` 4. If yes, to run a specific query, make a `POST` request (using `curl`, [Postman](https://www.getpostman.com/downloads/), Python `requests`, etc.) to the above endpoint with a `json` payload. For example, to search for all courses, run a query with Content-Type as `application/json` and with a body `{"query":{"term":{"object_type":"course"}}}` +### Running OpenSearch as a multi-node local cluster + +By default the configuration runs OpenSearch in `single-node` mode. If you'd like to run a 3-node cluster locally you can set the following environment variable in your shell. + +```shell +export OPENSEARCH_CLUSTER_TYPE=cluster +``` + +You should make this permanent by using `direnv` or similar so that all your shell sessions are using the same docker compose config): + +After setting this and running `docker compose up` you'll see this 3 node cluster be created. Note that the volumes used by these containers are separate from the volume used by the single-node setup so you will need to recreate your indicies. This is intentional and critical to being able to switch back and forth between `single-node` and `cluster` setups. + ### Running the app in a notebook This repo includes a config for running a [Jupyter notebook](https://jupyter.org/) in a Docker container. This enables you to do in a Jupyter notebook anything you might otherwise do in a Django shell. To get started: diff --git a/docker-compose.apps.yml b/docker-compose.apps.yml new file mode 100644 index 0000000000..06049cfe6b --- /dev/null +++ b/docker-compose.apps.yml @@ -0,0 +1,68 @@ +include: + - docker-compose.opensearch.${OPENSEARCH_CLUSTER_TYPE:-single-node}.yml + +services: + web: + profiles: + - backend + build: + context: . + dockerfile: Dockerfile + extends: + file: docker-compose.opensearch.${OPENSEARCH_CLUSTER_TYPE:-single-node}.apps.yml + service: web + mem_limit: 1gb + cpus: 2 + command: ./scripts/run-django-dev.sh + stdin_open: true + tty: true + ports: + - "8061:8061" + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - .:/src + - django_media:/var/media + + watch: + profiles: + - frontend + working_dir: /src + image: node:20.17 + entrypoint: ["/bin/sh", "-c"] + command: + - | + yarn install --immutable + yarn workspace mit-learn storybook --no-open & + yarn workspace mit-learn watch:docker + ports: + - "8062:8062" + - "6006:6006" + volumes: + - .:/src + + celery: + profiles: + - backend + build: + context: . + dockerfile: Dockerfile + extends: + file: docker-compose.opensearch.${OPENSEARCH_CLUSTER_TYPE:-single-node}.apps.yml + service: web + command: > + /bin/bash -c ' + sleep 3; + celery -A main.celery:app worker -Q default -B -l ${MITOL_LOG_LEVEL:-INFO} & + celery -A main.celery:app worker -Q edx_content,default -l ${MITOL_LOG_LEVEL:-INFO}' + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - .:/src + - django_media:/var/media diff --git a/docker-compose.codespaces.yml b/docker-compose.codespaces.yml index 8cfb5b6fdb..965f705a7e 100644 --- a/docker-compose.codespaces.yml +++ b/docker-compose.codespaces.yml @@ -1,90 +1,21 @@ -x-environment: &py-environment +include: + - docker-compose.services.yml services: - db: - extends: - file: docker-compose.yml - service: db - env_file: env/codespaces.env - redis: - extends: - file: docker-compose.yml - service: redis web: - profiles: - - backend - build: - context: . - dockerfile: Dockerfile - mem_limit: 1gb - cpus: 2 + extends: + file: docker-compose.apps.yml + service: web env_file: env/codespaces.env - environment: - PORT: 8061 - command: ./scripts/run-django-dev.sh - stdin_open: true - tty: true - ports: - - "8061:8061" - links: - - db - - opensearch-node-mitopen - - redis - volumes: - - .:/src - - django_media:/var/media + watch: - profiles: - - frontend - working_dir: /src - image: node:20.17 - entrypoint: ["/bin/sh", "-c"] - command: - - | - yarn install --immutable - yarn workspace mit-learn storybook --no-open & - yarn workspace mit-learn watch:docker + extends: + file: docker-compose.apps.yml + service: watch env_file: env/codespaces.env - ports: - - "8062:8062" - - "6006:6006" - volumes: - - .:/src + celery: - profiles: - - backend - build: - context: . - dockerfile: Dockerfile - env_file: env/codespaces.env - command: > - /bin/bash -c ' - sleep 3; - celery -A main.celery:app worker -Q default -B -l ${MITOL_LOG_LEVEL:-INFO} & - celery -A main.celery:app worker -Q edx_content,default -l ${MITOL_LOG_LEVEL:-INFO}' - links: - - db - - opensearch-node-mitopen - - redis - volumes: - - .:/src - - django_media:/var/media - tika: - extends: - file: docker-compose.yml - service: tika - opensearch-node-mitopen: extends: - file: docker-compose.yml - service: opensearch-node-mitopen + file: docker-compose.apps.yml + service: celery env_file: env/codespaces.env - nginx: - extends: - file: docker-compose.yml - service: nginx - -volumes: - opensearch-data1: - django_media: - yarn-cache: - pgdata: diff --git a/docker-compose.opensearch.base.yml b/docker-compose.opensearch.base.yml new file mode 100644 index 0000000000..a5602f6e6e --- /dev/null +++ b/docker-compose.opensearch.base.yml @@ -0,0 +1,21 @@ +services: + opensearch: + image: opensearchproject/opensearch:2.16.0 + environment: + - "cluster.name=opensearch-cluster" + - "bootstrap.memory_lock=true" # along with the memlock settings below, disables swapping + - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # Set min and max JVM heap sizes to at least 50% of system RAM + - "DISABLE_INSTALL_DEMO_CONFIG=true" # disables execution of install_demo_configuration.sh bundled with security plugin, which installs demo certificates and security configurations to OpenSearch + - "DISABLE_SECURITY_PLUGIN=true" # disables security plugin entirely in OpenSearch by setting plugins.security.disabled: true in opensearch.yml + healthcheck: + test: curl http://localhost:9200/_cluster/health || exit 1 + interval: 3s + timeout: 3s + retries: 20 + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems + hard: 65536 diff --git a/docker-compose.opensearch.cluster.apps.yml b/docker-compose.opensearch.cluster.apps.yml new file mode 100644 index 0000000000..cd473ffef1 --- /dev/null +++ b/docker-compose.opensearch.cluster.apps.yml @@ -0,0 +1,17 @@ +services: + web: + depends_on: + opensearch-node-mitopen-1: + condition: service_healthy + opensearch-node-mitopen-2: + condition: service_healthy + opensearch-node-mitopen-3: + condition: service_healthy + celery: + depends_on: + opensearch-node-mitopen-1: + condition: service_healthy + opensearch-node-mitopen-2: + condition: service_healthy + opensearch-node-mitopen-3: + condition: service_healthy diff --git a/docker-compose.opensearch.cluster.yml b/docker-compose.opensearch.cluster.yml new file mode 100644 index 0000000000..04021d814b --- /dev/null +++ b/docker-compose.opensearch.cluster.yml @@ -0,0 +1,48 @@ +services: + opensearch-node-mitopen-1: + extends: + file: docker-compose.opensearch.base.yml + service: opensearch + hostname: opensearch-node-mitopen-1 + environment: + - "node.name=opensearch-node-mitopen-1" + - "discovery.seed_hosts=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3" + - "cluster.initial_cluster_manager_nodes=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3" + volumes: + - opensearch-cluster-data1:/usr/share/opensearch/data + ports: + - 9100:9200 # REST API + - 9500:9600 # Performance Analyzer + opensearch-node-mitopen-2: + extends: + file: docker-compose.opensearch.base.yml + service: opensearch + hostname: opensearch-node-mitopen-2 + environment: + - "node.name=opensearch-node-mitopen-2" + - "discovery.seed_hosts=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3" + - "cluster.initial_cluster_manager_nodes=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3" + volumes: + - opensearch-cluster-data2:/usr/share/opensearch/data + ports: + - 9101:9200 # REST API + - 9501:9600 # Performance Analyzer + opensearch-node-mitopen-3: + extends: + file: docker-compose.opensearch.base.yml + service: opensearch + hostname: opensearch-node-mitopen-3 + environment: + - "node.name=opensearch-node-mitopen-3" + - "discovery.seed_hosts=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3" + - "cluster.initial_cluster_manager_nodes=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3" + volumes: + - opensearch-cluster-data3:/usr/share/opensearch/data + ports: + - 9102:9200 # REST API + - 9502:9600 # Performance Analyzer + +volumes: + opensearch-cluster-data1: + opensearch-cluster-data2: + opensearch-cluster-data3: diff --git a/docker-compose.opensearch.single-node.apps.yml b/docker-compose.opensearch.single-node.apps.yml new file mode 100644 index 0000000000..9a66e038f3 --- /dev/null +++ b/docker-compose.opensearch.single-node.apps.yml @@ -0,0 +1,9 @@ +services: + web: + depends_on: + opensearch-node-mitopen-1: + condition: service_healthy + celery: + depends_on: + opensearch-node-mitopen-1: + condition: service_healthy diff --git a/docker-compose.opensearch.single-node.yml b/docker-compose.opensearch.single-node.yml new file mode 100644 index 0000000000..c49b312c75 --- /dev/null +++ b/docker-compose.opensearch.single-node.yml @@ -0,0 +1,17 @@ +services: + opensearch-node-mitopen-1: + extends: + file: docker-compose.opensearch.base.yml + service: opensearch + hostname: opensearch-node-mitopen-1 + environment: + - "node.name=opensearch-node-mitopen-1" + - "discovery.type=single-node" # disables bootstrap checks that are enabled when network.host is set to a non-loopback address + volumes: + - opensearch-data1:/usr/share/opensearch/data + ports: + - 9100:9200 # REST API + - 9500:9600 # Performance Analyzer + +volumes: + opensearch-data1: diff --git a/docker-compose.services.yml b/docker-compose.services.yml new file mode 100644 index 0000000000..f43ce554ea --- /dev/null +++ b/docker-compose.services.yml @@ -0,0 +1,83 @@ +include: + - docker-compose.opensearch.${OPENSEARCH_CLUSTER_TYPE:-single-node}.yml + +services: + db: + profiles: + - backend + image: postgres:12.20 + healthcheck: + test: ["CMD", "pg_isready"] + interval: 3s + timeout: 3s + retries: 10 + ports: + - 5432:5432 + environment: + - PGUSER=postgres + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + volumes: + - pgdata:/var/lib/postgresql + + redis: + profiles: + - backend + image: redis:7.4.0 + healthcheck: + test: ["CMD", "redis-cli", "ping", "|", "grep", "PONG"] + interval: 3s + timeout: 3s + retries: 10 + ports: + - "6379" + + nginx: + profiles: + - backend + build: + context: ./nginx + ports: + - "8063:8063" + links: + - web + environment: + PORT: 8063 + NGINX_UWSGI_PASS: "web:8061" + volumes: + - ./config:/etc/nginx/templates + + tika: + profiles: + - backend + image: apache/tika:2.5.0 + ports: + - "9998:9998" + + locust: + image: locustio/locust + ports: + - "8089:8089" + volumes: + - ./load_testing:/mnt/locust + command: -f /mnt/locust/locustfile.py --master -H http://nginx:8063 --class-picker + links: + - nginx + profiles: + - load-testing + + locust-worker: + image: locustio/locust + volumes: + - ./load_testing:/mnt/locust + command: -f /mnt/locust/locustfile.py --worker --master-host locust + links: + - nginx + profiles: + - load-testing + +volumes: + pgdata: + # note: these are here instead of docker-compose.apps.yml because `extends` doesn't pull them in + django_media: + yarn-cache: diff --git a/docker-compose.yml b/docker-compose.yml index 7d9e91625a..584fe7c06c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,72 +1,11 @@ -x-environment: &py-environment +include: + - docker-compose.services.yml services: - db: - profiles: - - backend - image: postgres:12.20 - ports: - - 5432:5432 - environment: - - POSTGRES_PASSWORD=postgres - volumes: - - pgdata:/var/lib/postgresql - - redis: - profiles: - - backend - image: redis:7.4.0 - ports: - - "6379" - - opensearch-node-mitopen: - profiles: - - backend - image: opensearchproject/opensearch:2.16.0 - container_name: opensearch-node-mitopen - environment: - - cluster.name=opensearch-cluster - - node.name=opensearch-node-mitopen - - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping - - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # Set min and max JVM heap sizes to at least 50% of system RAM - - "DISABLE_INSTALL_DEMO_CONFIG=true" # disables execution of install_demo_configuration.sh bundled with security plugin, which installs demo certificates and security configurations to OpenSearch - - "DISABLE_SECURITY_PLUGIN=true" # disables security plugin entirely in OpenSearch by setting plugins.security.disabled: true in opensearch.yml - - "discovery.type=single-node" # disables bootstrap checks that are enabled when network.host is set to a non-loopback address - ulimits: - memlock: - soft: -1 - hard: -1 - nofile: - soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems - hard: 65536 - volumes: - - opensearch-data1:/usr/share/opensearch/data - ports: - - 9100:9200 - - nginx: - profiles: - - backend - build: - context: ./nginx - ports: - - "8063:8063" - links: - - web - environment: - PORT: 8063 - NGINX_UWSGI_PASS: "web:8061" - volumes: - - ./config:/etc/nginx/templates - web: - profiles: - - backend - build: - context: . - dockerfile: Dockerfile - mem_limit: 1gb - cpus: 2 + extends: + file: docker-compose.apps.yml + service: web env_file: - path: env/shared.env - path: env/shared.local.env @@ -77,30 +16,11 @@ services: # DEPRECATED: legacy .env file at the repo root - path: .env required: false - command: ./scripts/run-django-dev.sh - stdin_open: true - tty: true - ports: - - "8061:8061" - links: - - db - - opensearch-node-mitopen - - redis - volumes: - - .:/src - - django_media:/var/media watch: - profiles: - - frontend - working_dir: /src - image: node:20.17 - entrypoint: ["/bin/sh", "-c"] - command: - - | - yarn install --immutable - yarn workspace mit-learn storybook --no-open & - yarn workspace mit-learn watch:docker + extends: + file: docker-compose.apps.yml + service: watch env_file: - path: env/shared.env - path: env/shared.local.env @@ -111,18 +31,11 @@ services: # DEPRECATED: legacy .env file at the repo root - path: .env required: false - ports: - - "8062:8062" - - "6006:6006" - volumes: - - .:/src celery: - profiles: - - backend - build: - context: . - dockerfile: Dockerfile + extends: + file: docker-compose.apps.yml + service: celery env_file: - path: env/shared.env - path: env/shared.local.env @@ -133,49 +46,3 @@ services: # DEPRECATED: legacy .env file at the repo root - path: .env required: false - command: > - /bin/bash -c ' - sleep 3; - celery -A main.celery:app worker -Q default -B -l ${MITOL_LOG_LEVEL:-INFO} & - celery -A main.celery:app worker -Q edx_content,default -l ${MITOL_LOG_LEVEL:-INFO}' - links: - - db - - opensearch-node-mitopen - - redis - volumes: - - .:/src - - django_media:/var/media - tika: - profiles: - - backend - image: apache/tika:2.5.0 - ports: - - "9998:9998" - - locust: - image: locustio/locust - ports: - - "8089:8089" - volumes: - - ./load_testing:/mnt/locust - command: -f /mnt/locust/locustfile.py --master -H http://nginx:8063 --class-picker - links: - - nginx - profiles: - - load-testing - - locust-worker: - image: locustio/locust - volumes: - - ./load_testing:/mnt/locust - command: -f /mnt/locust/locustfile.py --worker --master-host locust - links: - - nginx - profiles: - - load-testing - -volumes: - opensearch-data1: - django_media: - yarn-cache: - pgdata: diff --git a/env/backend.env b/env/backend.env index e3815cdcef..a82e14ceba 100644 --- a/env/backend.env +++ b/env/backend.env @@ -22,7 +22,7 @@ MITOL_DB_DISABLE_SSL=True MITOL_FEATURES_DEFAULT=True MITOL_SECURE_SSL_REDIRECT=False -OPENSEARCH_URL=opensearch-node-mitopen:9200 +OPENSEARCH_URL=opensearch-node-mitopen-1:9200 OPENSEARCH_INDEX=discussions_local OPENSEARCH_INDEXING_CHUNK_SIZE=100 diff --git a/env/ci.env b/env/ci.env index 58a36aa3a2..f3f22bc430 100644 --- a/env/ci.env +++ b/env/ci.env @@ -5,7 +5,7 @@ DATABASE_URL=postgres://postgres:postgres@db:5432/e2e_postgres # pragma: allowli MITOL_SECURE_SSL_REDIRECT=False MITOL_DB_DISABLE_SSL=True MITOL_FEATURES_DEFAULT=True -OPENSEARCH_URL=opensearch-node-mitopen:9200 +OPENSEARCH_URL=opensearch-node-mitopen-1:9200 CELERY_TASK_ALWAYS_EAGER=False CELERY_BROKER_URL=redis://redis:6379/4 CELERY_RESULT_BACKEND=redis://redis:6379/4 diff --git a/env/codespaces.env b/env/codespaces.env index 590a225186..5ddce03263 100644 --- a/env/codespaces.env +++ b/env/codespaces.env @@ -2,7 +2,7 @@ MITOL_SUPPORT_EMAIL=support@localhost POSTHOG_TIMEOUT_MS=1500 MAILGUN_KEY=test DEBUG=False -OPENSEARCH_URL=opensearch-node-mitopen:9200 +OPENSEARCH_URL=opensearch-node-mitopen-1:9200 OPENSEARCH_INDEX=discussions_local OPENSEARCH_INDEXING_CHUNK_SIZE=100 MAILGUN_RECIPIENT_OVERRIDE=