Merge main into nextjs

.devcontainer/devcontainer.json

@@ -22,6 +22,6 @@
     "SEE_API_CLIENT_SECRET": {}
   },
   "features": {},
-  "postStartCommand": "while [ \"$(python manage.py showmigrations | grep \"\\[ \\]\" | wc -l)\" -ne \"0\" ]; do echo \"waiting for migrations\"; sleep 2; done && python manage.py update_offered_by && python manage.py update_platforms && python manage.py update_departments_schools && python manage.py backpopulate_resource_channels --overwrite --all && python manage.py update_course_number_departments && python manage.py backpopulate_mitxonline_data && python manage.py backpopulate_micromasters_data && python manage.py recreate_index --all",
+  "postStartCommand": "while [ \"$(python manage.py showmigrations | grep \"\\[ \\]\" | wc -l)\" -ne \"0\" ]; do echo \"waiting for migrations\"; sleep 2; done && python manage.py update_offered_by && python manage.py update_platforms && python manage.py update_departments_schools && python manage.py update_course_number_departments && python manage.py backpopulate_mitxonline_data && python manage.py backpopulate_micromasters_data && python manage.py backpopulate_resource_channels --overwrite --all && python manage.py recreate_index --all",
   "forwardPorts": [8062, 8063]
 }

.github/workflows/ci.yml

@@ -156,7 +156,6 @@ jobs:
           file: coverage/lcov.info
 
   build-deploy-frontend:
-    if: github.ref == 'refs/heads/nextjs'
     needs: javascript-tests
     runs-on: ubuntu-latest
     steps:
@@ -178,14 +177,33 @@ jobs:
           SUPPORT_EMAIL: mitlearn-support@mit.edu
           MITOL_AXIOS_WITH_CREDENTIALS: true
           CSRF_COOKIE_NAME: learn-rc-csrftoken
+          NEXT_PUBLIC_SENTRY_DSN: ${{ secrets.SENTRY_DSN_RC }}
+          NEXT_PUBLIC_SENTRY_ENV: rc
+          NEXT_PUBLIC_SENTRY_PROFILES_SAMPLE_RATE: 0.0
+          NEXT_PUBLIC_SENTRY_TRACES_SAMPLE_RATE: 0.0
+          NEXT_PUBLIC_VERSION: ${{ github.sha }}
+          NEXT_PUBLIC_POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY_RC }}
         run: |
           heroku container:push web \
           --app mitopen-rc-nextjs \
           --recursive \
-          --arg NEXT_PUBLIC_ORIGIN=$ORIGIN,NEXT_PUBLIC_MITOL_API_BASE_URL=$MITOL_API_BASE_URL,NEXT_PUBLIC_SITE_NAME="$SITE_NAME",NEXT_PUBLIC_MITOL_SUPPORT_EMAIL=$SUPPORT_EMAIL,NEXT_PUBLIC_EMBEDLY_KEY=$EMBEDLY_KEY,NEXT_PUBLIC_MITOL_AXIOS_WITH_CREDENTIALS=$MITOL_AXIOS_WITH_CREDENTIALS,NEXT_PUBLIC_CSRF_COOKIE_NAME=$CSRF_COOKIE_NAME \
+          --arg NEXT_PUBLIC_ORIGIN=$ORIGIN,\
+          NEXT_PUBLIC_MITOL_API_BASE_URL=$MITOL_API_BASE_URL,\
+          NEXT_PUBLIC_SITE_NAME="$SITE_NAME",\
+          NEXT_PUBLIC_MITOL_SUPPORT_EMAIL=$SUPPORT_EMAIL,\
+          NEXT_PUBLIC_EMBEDLY_KEY=$EMBEDLY_KEY,\
+          NEXT_PUBLIC_MITOL_AXIOS_WITH_CREDENTIALS=$MITOL_AXIOS_WITH_CREDENTIALS,\
+          NEXT_PUBLIC_CSRF_COOKIE_NAME=$CSRF_COOKIE_NAME,\
+          NEXT_PUBLIC_VERSION=$NEXT_PUBLIC_VERSION,\
+          NEXT_PUBLIC_SENTRY_DSN=$NEXT_PUBLIC_SENTRY_DSN,\
+          NEXT_PUBLIC_SENTRY_ENV=$NEXT_PUBLIC_SENTRY_ENV,\
+          NEXT_PUBLIC_SENTRY_PROFILES_SAMPLE_RATE=$NEXT_PUBLIC_SENTRY_PROFILES_SAMPLE_RATE,\
+          NEXT_PUBLIC_SENTRY_TRACES_SAMPLE_RATE=$NEXT_PUBLIC_SENTRY_TRACES_SAMPLE_RATE,\
+          NEXT_PUBLIC_POSTHOG_API_KEY=$NEXT_PUBLIC_POSTHOG_API_KEY \
           --context-path .
 
       - name: Release on Heroku
+        if: github.ref == 'refs/heads/nextjs'
         env:
           HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
         run: heroku container:release --app mitopen-rc-nextjs web

.yarnrc.yml

@@ -4,7 +4,7 @@ enableGlobalCache: false
 
 nodeLinker: node-modules
 
-yarnPath: .yarn/releases/yarn-4.4.1.cjs
+yarnPath: .yarn/releases/yarn-4.5.0.cjs
 
 # https://github.com/vitejs/vite-plugin-react-swc/issues/74#issuecomment-1520484130
 # https://github.com/swc-project/swc/issues/5616#issuecomment-1265639797

RELEASE.rst

@@ -1,6 +1,44 @@
 Release Notes
 =============
 
+Version 0.22.1 (Released October 15, 2024)
+--------------
+
+- increase learning path listing limit to 100 (#1692)
+- Shanbady/consolidate static fixtures (#1684)
+
+Version 0.22.0 (Released October 15, 2024)
+--------------
+
+- Make add to list dialog scrollable (#1689)
+- make search defaults settable (#1681)
+- Shuffling around where the search_update event is fired so it happens in more places (#1679)
+- prevent featured course carousel from re-randomizing (#1673)
+
+Version 0.21.3 (Released October 15, 2024)
+--------------
+
+- removing invalid flag from clear cache command (#1675)
+- Clear Cache on Deploy (#1668)
+- Update Yarn to v4.5.0 (#1624)
+- topic detail banner / subtopic logic revisions (#1646)
+- Restore program letter intercept view (#1643)
+
+Version 0.21.2 (Released October 10, 2024)
+--------------
+
+- Content File Score Adjustment (#1667)
+- remove unnecessary padding adjustment on Input base styles (#1666)
+
+Version 0.21.1 (Released October 09, 2024)
+--------------
+
+- Always delete past events during ETL, and filter them out from api results too just in case (#1660)
+- Add location field to LearningResource and LearningResourceRun models (#1604)
+- Cached department and topic page counts (#1661)
+- Shanbady/randomize featured resources (#1653)
+- Ignore NotFoundErrors in switch_indices function (#1654)
+
 Version 0.21.0 (Released October 07, 2024)
 --------------
 

app.json

@@ -91,6 +91,10 @@
       "description": "Default max incompleteness penalty value for the search API and frontend",
       "required": false
     },
+    "DEFAULT_SEARCH_CONTENT_FILE_SCORE_WEIGHT": {
+      "description": "Default score weight for content file search match",
+      "required": false
+    },
     "EDX_API_ACCESS_TOKEN_URL": {
       "description": "URL to retrieve a MITx access token",
       "required": false

data_fixtures/migrations/0001_add_testimonial_data.py

@@ -7,6 +7,16 @@
 from django.core.files import File
 from django.db import migrations
 
+from data_fixtures.utils import (
+    upsert_department_data,
+    upsert_offered_by_data,
+    upsert_platform_data,
+    upsert_school_data,
+    upsert_topic_data_file,
+)
+
+logger = logging.getLogger(__name__)
+
 """
 Fix an issue with PIL's logger when running in test
 https://github.com/camptocamp/pytest-odoo/issues/15
@@ -216,6 +226,27 @@
 ]
 
 
+def load_initial_fixtures(apps, schema_editor):
+    """
+    Load initial static fixtures required by
+    management commands further down
+    """
+    offerors = upsert_offered_by_data()
+    departments = upsert_department_data()
+    schools = upsert_school_data()
+    platforms = upsert_platform_data()
+    topics = upsert_topic_data_file()
+    logout = (
+        f"Updated:"
+        f"   {offerors} offerors"
+        f"   {departments} departments"
+        f"   {schools} schools"
+        f"   {platforms} platforms"
+        f"   {topics} topics"
+    )
+    logger.info(logout)
+
+
 def load_fixtures(apps, schema_editor):
     """
     Load fixtures for testimonials
@@ -252,5 +283,6 @@ class Migration(migrations.Migration):
     dependencies = []
 
     operations = [
+        migrations.RunPython(load_initial_fixtures, migrations.RunPython.noop),
         migrations.RunPython(load_fixtures, migrations.RunPython.noop),
     ]

data_fixtures/migrations/0004_upsert_initial_topic_data.py

@@ -2,7 +2,7 @@
 
 from django.db import migrations
 
-from learning_resources.utils import upsert_topic_data_file
+from data_fixtures.utils import upsert_topic_data_file
 
 
 def perform_topic_upsert(apps, schema_editor):

data_fixtures/migrations/0007_topic_mappings_edx_add_programming_coding_to_computer_science.py

@@ -5,7 +5,7 @@
 
 from django.db import migrations
 
-from learning_resources.utils import upsert_topic_data_string
+from data_fixtures.utils import upsert_topic_data_string
 
 map_changes = """
 ---

data_fixtures/migrations/0009_topics_update_engineering_add_manufacturing.py

@@ -8,7 +8,7 @@
 from django.db import migrations
 
 from channels.constants import ChannelType
-from learning_resources.utils import upsert_topic_data_string
+from data_fixtures.utils import upsert_topic_data_string
 
 map_changes = """
 ---

data_fixtures/migrations/0012_topic_mappings_adjust_sustainable_business_finance_accounting.py

@@ -7,7 +7,7 @@
 
 from django.db import migrations
 
-from learning_resources.utils import upsert_topic_data_string
+from data_fixtures.utils import upsert_topic_data_string
 
 forward_map_changes = """
 ---

data_fixtures/migrations/0014_add_department_SP.py

@@ -33,6 +33,10 @@ def add_sp(apps, schema_editor):
     department, _ = LearningResourceDepartment.objects.get_or_create(
         department_id="SP",
         name="Special Programs",
+        defaults={
+            "department_id": "SP",
+            "name": "Special Programs",
+        },
     )
     Channel = apps.get_model("channels", "Channel")
 
@@ -43,7 +47,9 @@ def add_sp(apps, schema_editor):
         title=department.name,
     )
     ChannelDepartmentDetail.objects.get_or_create(
-        channel=channel, department=department
+        channel=channel,
+        department=department,
+        defaults={"channel": channel, "department": department},
     )
 
 

data_fixtures/utils.py

@@ -0,0 +1,147 @@
+import json
+from pathlib import Path
+
+import yaml
+from django.db import transaction
+
+from learning_resources.models import (
+    LearningResourceDepartment,
+    LearningResourceOfferor,
+    LearningResourcePlatform,
+    LearningResourceSchool,
+)
+from learning_resources.utils import (
+    _walk_topic_map,
+    department_delete_actions,
+    offeror_delete_actions,
+    offeror_upserted_actions,
+)
+
+
+@transaction.atomic()
+def upsert_topic_data_file(
+    config_path: str = "learning_resources/data/topics.yaml",
+) -> None:
+    """
+    Load the topics from a yaml file.
+
+    The yaml file should have a root "topics" key, and then any number of topic
+    records beneath it. See _walk_topic_map for an explanation of the record
+    format.
+
+    Args:
+    - config_path (str): the path to the topics file.
+    Returns:
+    - None
+    """
+
+    with Path.open(Path(config_path)) as topic_file:
+        topic_file_yaml = topic_file.read()
+
+    topics = yaml.safe_load(topic_file_yaml)
+
+    _walk_topic_map(topics["topics"])
+
+
+@transaction.atomic()
+def upsert_offered_by_data():
+    """
+    Upsert LearningResourceOfferor data
+    """
+    offerors = []
+    with Path.open(Path(__file__).parent / "fixtures" / "offered_by.json") as inf:
+        offered_by_json = json.load(inf)
+        for offeror in offered_by_json:
+            offeror_fields = offeror["fields"]
+            offered_by, _ = LearningResourceOfferor.objects.update_or_create(
+                code=offeror_fields["code"],
+                defaults=offeror_fields,
+            )
+            offeror_upserted_actions(offered_by, overwrite=True)
+            offerors.append(offeror_fields["name"])
+        invalid_offerors = LearningResourceOfferor.objects.exclude(name__in=offerors)
+        for offeror in invalid_offerors:
+            offeror_delete_actions(offeror)
+    return offerors
+
+
+@transaction.atomic()
+def upsert_department_data():
+    """
+    Upsert LearningResourceDepartment data
+    """
+    departments = []
+    with Path.open(Path(__file__).parent / "fixtures" / "departments.json") as inf:
+        departments_json = json.load(inf)
+        for dept in departments_json:
+            department_fields = dept["fields"]
+            LearningResourceDepartment.objects.update_or_create(
+                department_id=department_fields["department_id"],
+                defaults=department_fields,
+            )
+            departments.append(department_fields["name"])
+        invalid_departments = LearningResourceDepartment.objects.exclude(
+            name__in=departments
+        ).all()
+        for invalid_department in invalid_departments:
+            department_delete_actions(invalid_department)
+    return departments
+
+
+@transaction.atomic()
+def upsert_school_data():
+    """
+    Upsert LearningResourceSchool data
+    """
+    schools = []
+    with Path.open(Path(__file__).parent / "fixtures" / "schools.json") as inf:
+        schools_json = json.load(inf)
+        for school in schools_json:
+            school_fields = school["fields"]
+            LearningResourceSchool.objects.update_or_create(
+                id=school_fields["id"],
+                defaults=school_fields,
+            )
+            schools.append(school_fields["name"])
+        LearningResourceSchool.objects.exclude(name__in=schools).delete()
+    return schools
+
+
+@transaction.atomic()
+def upsert_platform_data():
+    """
+    Upsert LearningResourcePlatform data
+    """
+    platforms = []
+    with Path.open(Path(__file__).parent / "fixtures" / "platforms.json") as inf:
+        platform_json = json.load(inf)
+        for platform in platform_json:
+            platform_fields = platform["fields"]
+            LearningResourcePlatform.objects.update_or_create(
+                code=platform_fields["code"],
+                defaults=platform_fields,
+            )
+            platforms.append(platform_fields["code"])
+        LearningResourcePlatform.objects.exclude(code__in=platforms).delete()
+    return platforms
+
+
+@transaction.atomic()
+def upsert_topic_data_string(yaml_data: str) -> None:
+    """
+    Load the topics from a yaml string.
+
+    The yaml string should be formatted in the same way that it is for
+    upsert_topic_data_file - this function exists just to allow you to specify
+    the data as a string so you can roll it into a migration file in the
+    data_fixtures app.
+
+    Args:
+    - yaml_data (str): the yaml to process
+    Returns:
+    - None
+    """
+
+    topics = yaml.safe_load(yaml_data)
+
+    _walk_topic_map(topics["topics"])

frontends/api/src/clients.ts

@@ -20,6 +20,7 @@ import {
   NewsEventsApi,
   ProfilesApi,
   TestimonialsApi,
+  LearningResourcesSearchAdminParamsApi,
 } from "./generated/v0/api"
 
 import axiosInstance from "./axios"
@@ -46,6 +47,9 @@ const learningResourcesSearchApi = new LearningResourcesSearchApi(
   axiosInstance,
 )
 
+const learningResourcesSearchAdminParamsApi =
+  new LearningResourcesSearchAdminParamsApi(undefined, BASE_PATH, axiosInstance)
+
 const featuredApi = new FeaturedApi(undefined, BASE_PATH, axiosInstance)
 
 const learningpathsApi = new LearningpathsApi(
@@ -108,4 +112,5 @@ export {
   newsEventsApi,
   featuredApi,
   testimonialsApi,
+  learningResourcesSearchAdminParamsApi,
 }