Release 0.24.1

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ ci:
     - style-lint
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -41,7 +41,7 @@ repos:
             html,
           ]
   - repo: https://github.com/scop/pre-commit-shfmt
-    rev: v3.9.0-1
+    rev: v3.10.0-1
     hooks:
       - id: shfmt
   - repo: https://github.com/adrienverge/yamllint.git
@@ -74,7 +74,7 @@ repos:
           - ".*/generated/"
         additional_dependencies: ["gibberish-detector"]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.6.7"
+    rev: "v0.7.1"
     hooks:
       - id: ruff-format
       - id: ruff

RELEASE.rst

@@ -1,6 +1,15 @@
 Release Notes
 =============
 
+Version 0.24.1
+--------------
+
+- Fix prolearn bug for existing resources whose urls have changed (#1775)
+- [pre-commit.ci] pre-commit autoupdate (#1629)
+- Update typescript-eslint monorepo to v8 (#1534)
+- fixing intermittent test failure (#1773)
+- Professional Education ETL Pipeline (#1210)
+
 Version 0.24.0 (Released October 29, 2024)
 --------------
 

app.json

@@ -229,6 +229,10 @@
       "description": "URL to MicroMasters catalog API",
       "required": "false"
     },
+    "MITPE_API_ENABLED": {
+      "description": "Whether MIT Professional Education ETL should be enabled",
+      "required": "false"
+    },
     "MITPE_BASE_URL": {
       "description": "Base URL for MIT Professional Education website",
       "required": "false"
@@ -511,10 +515,6 @@
       "description": "URL to retrieve a MITx access token",
       "required": false
     },
-    "SEE_API_ENABLED": {
-      "description": "Whether the Sloan ETL shouold be enabled",
-      "required": false
-    },
     "SEE_API_URL": {
       "description": "URL to retrieve MITx course data from",
       "required": false

frontends/package.json

@@ -47,8 +47,8 @@
     "@types/jest": "^29.5.2",
     "@types/jest-when": "^3.5.2",
     "@types/react-infinite-scroller": "^1.2.3",
-    "@typescript-eslint/eslint-plugin": "^7.0.0",
-    "@typescript-eslint/typescript-estree": "^5.59.2",
+    "@typescript-eslint/eslint-plugin": "^8.0.0",
+    "@typescript-eslint/typescript-estree": "^8.0.0",
     "cross-fetch": "^4.0.0",
     "eslint": "8.57.1",
     "eslint-config-mitodl": "^2.1.0",

learning_resources/etl/constants.py

@@ -52,13 +52,14 @@ class ETLSource(ExtendedEnum):
 
     micromasters = "micromasters"
     mit_edx = "mit_edx"
+    mitpe = "mitpe"
     mitxonline = "mitxonline"
     oll = "oll"
-    xpro = "xpro"
     ocw = "ocw"
     prolearn = "prolearn"
     podcast = "podcast"
     see = "see"
+    xpro = "xpro"
     youtube = "youtube"
 
 
@@ -82,6 +83,9 @@ class CourseNumberType(Enum):
     "": LearningResourceDelivery.online.name,
     "Blended": LearningResourceDelivery.hybrid.name,
     "In Person": LearningResourceDelivery.in_person.name,
+    "Live Virtual": LearningResourceDelivery.online.name,
+    "Live Online": LearningResourceDelivery.online.name,
+    "On Campus": LearningResourceDelivery.in_person.name,
     **{
         value: LearningResourceDelivery(value).name
         for value in LearningResourceDelivery.values()

learning_resources/etl/loaders.py

@@ -301,7 +301,7 @@ def load_run(
     return learning_resource_run
 
 
-def upsert_course_or_program(
+def upsert_course_or_program(  # noqa: C901
     resource_data: dict,
     blocklist: list[str],
     duplicates: list[dict],
@@ -388,9 +388,25 @@ def upsert_course_or_program(
             platform=platform,
             resource_type=LearningResourceType.course.name,
         ).order_by("-updated_on")
-        if existing_courses.count() > 1:
+        if existing_courses.count() > 0:
             for course in existing_courses[1:]:
                 resource_delete_actions(course)
+        # does a resource with the same readable id already exist?
+        # if so, update it with the new unique field value
+        elif LearningResource.objects.filter(
+            readable_id=resource_data["readable_id"],
+            platform=platform,
+            resource_type=LearningResourceType.course.name,
+        ).exists():
+            resource_data[unique_field_name] = unique_field_value
+            unique_field_name = READABLE_ID_FIELD
+            unique_field_value = resource_data.pop(READABLE_ID_FIELD)
+        return LearningResource.objects.select_for_update().update_or_create(
+            **{unique_field_name: unique_field_value},
+            platform=platform,
+            resource_type=resource_type,
+            defaults=resource_data,
+        )
     else:
         unique_field_value = resource_id
     return LearningResource.objects.select_for_update().update_or_create(

learning_resources/etl/loaders_test.py

@@ -627,6 +627,46 @@ def test_load_course_unique_urls(unique_url):
     assert old_course == result
 
 
+def test_load_course_old_id_new_url():
+    """
+    If url is supposed to be unique field, and a resource with the same readable_id
+    but different url exists, that resource should be updated with the new url.
+    """
+    unique_url = "https://mit.edu/unique.html"
+    readable_id = "new_unique_course_id"
+    platform = LearningResourcePlatformFactory.create(code=PlatformType.ocw.name)
+    existing_course = LearningResourceFactory.create(
+        readable_id=readable_id,
+        url="https://mit.edu/old.html",
+        platform=platform,
+        is_course=True,
+    )
+    props = {
+        "readable_id": readable_id,
+        "platform": PlatformType.ocw.name,
+        "offered_by": {"code": OfferedBy.ocw.name},
+        "title": "New title",
+        "url": unique_url,
+        "description": "something",
+        "unique_field": "url",
+        "runs": [
+            {
+                "run_id": "run_id",
+                "enrollment_start": "2024-01-01T00:00:00Z",
+                "start_date": "2024-01-20T00:00:00Z",
+                "end_date": "2024-06-20T00:00:00Z",
+            }
+        ],
+    }
+    result = load_course(props, [], [])
+    assert result.readable_id == readable_id
+    assert result.url == unique_url
+    assert result.published is True
+    existing_course.refresh_from_db()
+    assert existing_course.url == unique_url
+    assert existing_course.readable_id == readable_id
+
+
 @pytest.mark.parametrize("course_exists", [True, False])
 def test_load_course_fetch_only(mocker, course_exists):
     """When fetch_only is True, course should just be fetched from db"""