Release 0.19.0

.devcontainer/devcontainer.json

@@ -4,7 +4,7 @@
   "runServices": [
     "watch",
     "web",
-    "opensearch-node-mitopen",
+    "opensearch-node-mitopen-1",
     "db",
     "tika",
     "celery",

.github/workflows/ci.yml

@@ -28,7 +28,7 @@ jobs:
           - 6379:6379
 
       elastic:
-        image: docker.elastic.co/elasticsearch/elasticsearch:7.17.23
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.15.1
         env:
           network.host: "0.0.0.0"
           http.cors.enabled: "true"
@@ -145,7 +145,7 @@ jobs:
           NODE_ENV: test
 
       - name: Upload frontend build
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4
         with:
           name: frontend-build
           path: frontends/mit-learn/build
@@ -301,7 +301,7 @@ jobs:
 
       - name: Upload artifact
         if: always()
-        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4
         with:
           name: playwright-report
           path: e2e_testing/playwright-report

Dockerfile

@@ -56,6 +56,6 @@ RUN apt-get clean && apt-get purge
 
 USER mitodl
 
-EXPOSE 8063
-ENV PORT 8063
+EXPOSE 8061
+ENV PORT 8061
 CMD uwsgi uwsgi.ini

README.md

@@ -175,6 +175,18 @@ To enable searching the course catalog on opensearch, run through these steps:
 3. Once created and with `docker compose up` running, hit this endpoint in your browser to see if the index exists: `http://localhost:9101/discussions_local_all_default/_search`
 4. If yes, to run a specific query, make a `POST` request (using `curl`, [Postman](https://www.getpostman.com/downloads/), Python `requests`, etc.) to the above endpoint with a `json` payload. For example, to search for all courses, run a query with Content-Type as `application/json` and with a body `{"query":{"term":{"object_type":"course"}}}`
 
+### Running OpenSearch as a multi-node local cluster
+
+By default the configuration runs OpenSearch in `single-node` mode. If you'd like to run a 3-node cluster locally you can set the following environment variable in your shell.
+
+```shell
+export OPENSEARCH_CLUSTER_TYPE=cluster
+```
+
+You should make this permanent by using `direnv` or similar so that all your shell sessions are using the same docker compose config):
+
+After setting this and running `docker compose up` you'll see this 3 node cluster be created. Note that the volumes used by these containers are separate from the volume used by the single-node setup so you will need to recreate your indicies. This is intentional and critical to being able to switch back and forth between `single-node` and `cluster` setups.
+
 ### Running the app in a notebook
 
 This repo includes a config for running a [Jupyter notebook](https://jupyter.org/) in a Docker container. This enables you to do in a Jupyter notebook anything you might otherwise do in a Django shell. To get started:

RELEASE.rst

@@ -1,6 +1,27 @@
 Release Notes
 =============
 
+Version 0.19.0
+--------------
+
+- Update docker-compose config to support OS cluster (#1540)
+- Update dependency faker to v28 (#1550)
+- Update actions/upload-artifact digest to 5076954 (#1528)
+- Update dependency attrs to v24 (#1535)
+- Update dependency @faker-js/faker to v9 (#1533)
+- Update dependency pytest-cov to v5 (#1549)
+- Update docker.elastic.co/elasticsearch/elasticsearch Docker tag to v8 (#1260)
+- ETL pipeline for MIT edX programs (#1536)
+- license_cc and continuing_ed_credits fields (#1544)
+- Fix data migration erroring if a record exists (#1543)
+- learning resource drawer list buttons (#1537)
+- Add LearningResource.delivery field (eventually to replace learning_format) (#1510)
+- Update mcr.microsoft.com/playwright Docker tag to v1.47.0 (#1531)
+- Update dependency ruff to v0.6.4 (#1530)
+- Update dependency Django to v4.2.16 (#1529)
+- display sub topics on topic channels (#1511)
+- Sort news items by publish_date (#1522)
+
 Version 0.18.2 (Released September 12, 2024)
 --------------
 

app.json

@@ -103,6 +103,10 @@
       "description": "S3 prefix for MITx bucket keys",
       "required": false
     },
+    "EDX_PROGRAMS_API_URL": {
+      "description": "The catalog url for MITx programs",
+      "required": false
+    },
     "OPENSEARCH_HTTP_AUTH": {
       "description": "Basic auth settings for connecting to OpenSearch"
     },

data_fixtures/migrations/0014_add_department_SP.py

@@ -30,7 +30,7 @@ def add_sp(apps, schema_editor):
     Channel = apps.get_model("channels", "Channel")
     ChannelDepartmentDetail = apps.get_model("channels", "ChannelDepartmentDetail")
 
-    department = LearningResourceDepartment.objects.create(
+    department, _ = LearningResourceDepartment.objects.get_or_create(
         department_id="SP",
         name="Special Programs",
     )
@@ -42,7 +42,9 @@ def add_sp(apps, schema_editor):
         name=slugify(department.name),
         title=department.name,
     )
-    ChannelDepartmentDetail.objects.create(channel=channel, department=department)
+    ChannelDepartmentDetail.objects.get_or_create(
+        channel=channel, department=department
+    )
 
 
 class Migration(migrations.Migration):

docker-compose.apps.yml

@@ -0,0 +1,68 @@
+include:
+  - docker-compose.opensearch.${OPENSEARCH_CLUSTER_TYPE:-single-node}.yml
+
+services:
+  web:
+    profiles:
+      - backend
+    build:
+      context: .
+      dockerfile: Dockerfile
+    extends:
+      file: docker-compose.opensearch.${OPENSEARCH_CLUSTER_TYPE:-single-node}.apps.yml
+      service: web
+    mem_limit: 1gb
+    cpus: 2
+    command: ./scripts/run-django-dev.sh
+    stdin_open: true
+    tty: true
+    ports:
+      - "8061:8061"
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    volumes:
+      - .:/src
+      - django_media:/var/media
+
+  watch:
+    profiles:
+      - frontend
+    working_dir: /src
+    image: node:20.17
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        yarn install --immutable
+        yarn workspace mit-learn storybook --no-open &
+        yarn workspace mit-learn watch:docker
+    ports:
+      - "8062:8062"
+      - "6006:6006"
+    volumes:
+      - .:/src
+
+  celery:
+    profiles:
+      - backend
+    build:
+      context: .
+      dockerfile: Dockerfile
+    extends:
+      file: docker-compose.opensearch.${OPENSEARCH_CLUSTER_TYPE:-single-node}.apps.yml
+      service: web
+    command: >
+      /bin/bash -c '
+      sleep 3;
+      celery -A main.celery:app worker -Q default -B -l ${MITOL_LOG_LEVEL:-INFO} &
+      celery -A main.celery:app worker -Q edx_content,default -l ${MITOL_LOG_LEVEL:-INFO}'
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    volumes:
+      - .:/src
+      - django_media:/var/media

docker-compose.codespaces.yml

@@ -1,90 +1,21 @@
-x-environment: &py-environment
+include:
+  - docker-compose.services.yml
 
 services:
-  db:
-    extends:
-      file: docker-compose.yml
-      service: db
-    env_file: env/codespaces.env
-  redis:
-    extends:
-      file: docker-compose.yml
-      service: redis
   web:
-    profiles:
-      - backend
-    build:
-      context: .
-      dockerfile: Dockerfile
-    mem_limit: 1gb
-    cpus: 2
+    extends:
+      file: docker-compose.apps.yml
+      service: web
     env_file: env/codespaces.env
-    environment:
-      PORT: 8061
-    command: ./scripts/run-django-dev.sh
-    stdin_open: true
-    tty: true
-    ports:
-      - "8061:8061"
-    links:
-      - db
-      - opensearch-node-mitopen
-      - redis
-    volumes:
-      - .:/src
-      - django_media:/var/media
+
   watch:
-    profiles:
-      - frontend
-    working_dir: /src
-    image: node:20.17
-    entrypoint: ["/bin/sh", "-c"]
-    command:
-      - |
-        yarn install --immutable
-        yarn workspace mit-learn storybook --no-open &
-        yarn workspace mit-learn watch:docker
+    extends:
+      file: docker-compose.apps.yml
+      service: watch
     env_file: env/codespaces.env
-    ports:
-      - "8062:8062"
-      - "6006:6006"
-    volumes:
-      - .:/src
+
   celery:
-    profiles:
-      - backend
-    build:
-      context: .
-      dockerfile: Dockerfile
-    env_file: env/codespaces.env
-    command: >
-      /bin/bash -c '
-      sleep 3;
-      celery -A main.celery:app worker -Q default -B -l ${MITOL_LOG_LEVEL:-INFO} &
-      celery -A main.celery:app worker -Q edx_content,default -l ${MITOL_LOG_LEVEL:-INFO}'
-    links:
-      - db
-      - opensearch-node-mitopen
-      - redis
-    volumes:
-      - .:/src
-      - django_media:/var/media
-  tika:
-    extends:
-      file: docker-compose.yml
-      service: tika
-  opensearch-node-mitopen:
     extends:
-      file: docker-compose.yml
-      service: opensearch-node-mitopen
+      file: docker-compose.apps.yml
+      service: celery
     env_file: env/codespaces.env
-  nginx:
-    extends:
-      file: docker-compose.yml
-      service: nginx
-
-volumes:
-  opensearch-data1:
-  django_media:
-  yarn-cache:
-  pgdata:

docker-compose.opensearch.base.yml

@@ -0,0 +1,21 @@
+services:
+  opensearch:
+    image: opensearchproject/opensearch:2.16.0
+    environment:
+      - "cluster.name=opensearch-cluster"
+      - "bootstrap.memory_lock=true" # along with the memlock settings below, disables swapping
+      - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # Set min and max JVM heap sizes to at least 50% of system RAM
+      - "DISABLE_INSTALL_DEMO_CONFIG=true" # disables execution of install_demo_configuration.sh bundled with security plugin, which installs demo certificates and security configurations to OpenSearch
+      - "DISABLE_SECURITY_PLUGIN=true" # disables security plugin entirely in OpenSearch by setting plugins.security.disabled: true in opensearch.yml
+    healthcheck:
+      test: curl http://localhost:9200/_cluster/health || exit 1
+      interval: 3s
+      timeout: 3s
+      retries: 20
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
+        hard: 65536

docker-compose.opensearch.cluster.apps.yml

@@ -0,0 +1,17 @@
+services:
+  web:
+    depends_on:
+      opensearch-node-mitopen-1:
+        condition: service_healthy
+      opensearch-node-mitopen-2:
+        condition: service_healthy
+      opensearch-node-mitopen-3:
+        condition: service_healthy
+  celery:
+    depends_on:
+      opensearch-node-mitopen-1:
+        condition: service_healthy
+      opensearch-node-mitopen-2:
+        condition: service_healthy
+      opensearch-node-mitopen-3:
+        condition: service_healthy

docker-compose.opensearch.cluster.yml

@@ -0,0 +1,48 @@
+services:
+  opensearch-node-mitopen-1:
+    extends:
+      file: docker-compose.opensearch.base.yml
+      service: opensearch
+    hostname: opensearch-node-mitopen-1
+    environment:
+      - "node.name=opensearch-node-mitopen-1"
+      - "discovery.seed_hosts=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3"
+      - "cluster.initial_cluster_manager_nodes=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3"
+    volumes:
+      - opensearch-cluster-data1:/usr/share/opensearch/data
+    ports:
+      - 9100:9200 # REST API
+      - 9500:9600 # Performance Analyzer
+  opensearch-node-mitopen-2:
+    extends:
+      file: docker-compose.opensearch.base.yml
+      service: opensearch
+    hostname: opensearch-node-mitopen-2
+    environment:
+      - "node.name=opensearch-node-mitopen-2"
+      - "discovery.seed_hosts=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3"
+      - "cluster.initial_cluster_manager_nodes=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3"
+    volumes:
+      - opensearch-cluster-data2:/usr/share/opensearch/data
+    ports:
+      - 9101:9200 # REST API
+      - 9501:9600 # Performance Analyzer
+  opensearch-node-mitopen-3:
+    extends:
+      file: docker-compose.opensearch.base.yml
+      service: opensearch
+    hostname: opensearch-node-mitopen-3
+    environment:
+      - "node.name=opensearch-node-mitopen-3"
+      - "discovery.seed_hosts=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3"
+      - "cluster.initial_cluster_manager_nodes=opensearch-node-mitopen-1,opensearch-node-mitopen-2,opensearch-node-mitopen-3"
+    volumes:
+      - opensearch-cluster-data3:/usr/share/opensearch/data
+    ports:
+      - 9102:9200 # REST API
+      - 9502:9600 # Performance Analyzer
+
+volumes:
+  opensearch-cluster-data1:
+  opensearch-cluster-data2:
+  opensearch-cluster-data3:

docker-compose.opensearch.single-node.apps.yml

@@ -0,0 +1,9 @@
+services:
+  web:
+    depends_on:
+      opensearch-node-mitopen-1:
+        condition: service_healthy
+  celery:
+    depends_on:
+      opensearch-node-mitopen-1:
+        condition: service_healthy