From 6fb269f42c67e3177bb14c32a1cc685d49bd95f5 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Mon, 9 Sep 2024 14:57:08 -0400 Subject: [PATCH 1/3] ETL pipeline for MIT edX programs --- app.json | 4 + env/codespaces.env | 1 + learning_resources/etl/conftest.py | 8 + learning_resources/etl/mit_edx.py | 3 +- learning_resources/etl/mit_edx_programs.py | 72 + .../etl/mit_edx_programs_test.py | 13 + learning_resources/etl/openedx.py | 215 ++- learning_resources/etl/openedx_test.py | 172 +- learning_resources/etl/pipelines.py | 14 +- learning_resources/etl/pipelines_test.py | 33 +- learning_resources/tasks.py | 5 +- learning_resources/tasks_test.py | 5 +- main/settings_course_etl.py | 1 + test_json/test_mitx_programs.json | 1535 +++++++++++++++++ 14 files changed, 2036 insertions(+), 45 deletions(-) create mode 100644 learning_resources/etl/mit_edx_programs.py create mode 100644 learning_resources/etl/mit_edx_programs_test.py create mode 100644 test_json/test_mitx_programs.json diff --git a/app.json b/app.json index 9ddc3405dd..52b749d30b 100644 --- a/app.json +++ b/app.json @@ -103,6 +103,10 @@ "description": "S3 prefix for MITx bucket keys", "required": false }, + "EDX_PROGRAMS_API_URL": { + "description": "The catalog url for MITx programs", + "required": false + }, "OPENSEARCH_HTTP_AUTH": { "description": "Basic auth settings for connecting to OpenSearch" }, diff --git a/env/codespaces.env b/env/codespaces.env index ea84a2d1c7..590a225186 100644 --- a/env/codespaces.env +++ b/env/codespaces.env @@ -34,6 +34,7 @@ XPRO_CATALOG_API_URL=https://xpro.mit.edu/api/programs/ XPRO_COURSES_API_URL=https://xpro.mit.edu/api/courses/ EDX_API_ACCESS_TOKEN_URL=https://api.edx.org/oauth2/v1/access_token EDX_API_URL=https://api.edx.org/catalog/v1/catalogs/10/courses +EDX_PROGRAMS_API_URL=https://discovery.edx.org/api/v1/programs/ OCW_BASE_URL=https://ocw.mit.edu/ MICROMASTERS_CATALOG_API_URL=https://micromasters.mit.edu/api/v0/catalog/ MICROMASTERS_COURSE_URL=https://micromasters.mit.edu/api/v0/courseruns/ diff --git a/learning_resources/etl/conftest.py b/learning_resources/etl/conftest.py index 042719c531..12ce7de2ef 100644 --- a/learning_resources/etl/conftest.py +++ b/learning_resources/etl/conftest.py @@ -1,6 +1,7 @@ """Common ETL test fixtures""" import json +from pathlib import Path import pytest @@ -31,3 +32,10 @@ def non_mitx_course_data(): """Catalog data fixture""" with open("./test_json/test_non_mitx_course.json") as f: # noqa: PTH123 yield json.loads(f.read()) + + +@pytest.fixture +def mitx_programs_data(): + """Yield a data fixture for MITx programs""" + with Path.open(Path("./test_json/test_mitx_programs.json")) as f: + yield json.loads(f.read()) diff --git a/learning_resources/etl/mit_edx.py b/learning_resources/etl/mit_edx.py index cc2ebf1f68..4bd4da9de4 100644 --- a/learning_resources/etl/mit_edx.py +++ b/learning_resources/etl/mit_edx.py @@ -5,7 +5,7 @@ from django.conf import settings from toolz import compose, curried -from learning_resources.constants import OfferedBy, PlatformType +from learning_resources.constants import LearningResourceType, OfferedBy, PlatformType from learning_resources.etl.constants import ETLSource from learning_resources.etl.openedx import ( MIT_OWNER_KEYS, @@ -71,6 +71,7 @@ def get_open_edx_config(): PlatformType.edx.name, OfferedBy.mitx.name, ETLSource.mit_edx.name, + LearningResourceType.course.name, ) diff --git a/learning_resources/etl/mit_edx_programs.py b/learning_resources/etl/mit_edx_programs.py new file mode 100644 index 0000000000..006093f548 --- /dev/null +++ b/learning_resources/etl/mit_edx_programs.py @@ -0,0 +1,72 @@ +"""MIT edX ETL""" + +import logging + +from django.conf import settings +from toolz import compose, curried + +from learning_resources.constants import LearningResourceType, OfferedBy, PlatformType +from learning_resources.etl.constants import ETLSource +from learning_resources.etl.openedx import ( + MIT_OWNER_KEYS, + OpenEdxConfiguration, + openedx_extract_transform_factory, +) + +log = logging.getLogger() + + +def _is_mit_program(program): + """ + Helper function to determine if a course is an MIT course + + Args: + course (dict): The JSON object representing the course with all its course runs + + Returns: + bool: indicates whether the course is owned by MIT + """ # noqa: D401 + return ( + any( + owner["key"] in MIT_OWNER_KEYS + for owner in program.get("authoring_organizations") + ) + and "micromasters" not in program.get("type", "").lower() + and program.get("status") == "active" + ) + + +def get_open_edx_config(): + """ + Return the OpenEdxConfiguration for edX. + """ + required_settings = [ + "EDX_API_CLIENT_ID", + "EDX_API_CLIENT_SECRET", + "EDX_API_ACCESS_TOKEN_URL", + "EDX_PROGRAMS_API_URL", + "EDX_BASE_URL", + "EDX_ALT_URL", + ] + for setting in required_settings: + if not getattr(settings, setting): + log.warning("Missing required setting %s", setting) + return OpenEdxConfiguration( + settings.EDX_API_CLIENT_ID, + settings.EDX_API_CLIENT_SECRET, + settings.EDX_API_ACCESS_TOKEN_URL, + settings.EDX_PROGRAMS_API_URL, + settings.EDX_BASE_URL, + settings.EDX_ALT_URL, + PlatformType.edx.name, + OfferedBy.mitx.name, + ETLSource.mit_edx.name, + LearningResourceType.program.name, + ) + + +# use the OpenEdx factory to create our extract and transform funcs +extract, _transform = openedx_extract_transform_factory(get_open_edx_config) + +# modified transform function that filters the course list to ones that pass the _is_mit_course() predicate # noqa: E501 +transform = compose(_transform, curried.filter(_is_mit_program)) diff --git a/learning_resources/etl/mit_edx_programs_test.py b/learning_resources/etl/mit_edx_programs_test.py new file mode 100644 index 0000000000..d34e0e0626 --- /dev/null +++ b/learning_resources/etl/mit_edx_programs_test.py @@ -0,0 +1,13 @@ +"""Tests for mit_edx_programs""" + +import pytest + +from learning_resources.etl.mit_edx_programs import transform + + +@pytest.mark.django_db +def test_mitx_transform_mit_owner(mitx_programs_data): + """Verify that only non-micromasters programs with MIT owners are returned""" + transformed = list(transform(mitx_programs_data)) + assert len(transformed) == 1 + assert transformed[0]["title"] == "Circuits and Electronics" diff --git a/learning_resources/etl/openedx.py b/learning_resources/etl/openedx.py index 1cd24a44a1..ec1e83c7ef 100644 --- a/learning_resources/etl/openedx.py +++ b/learning_resources/etl/openedx.py @@ -7,6 +7,7 @@ import re from collections import namedtuple from datetime import UTC, datetime +from decimal import Decimal from pathlib import Path import requests @@ -18,6 +19,7 @@ Availability, CertificationType, LearningResourceType, + PlatformType, RunAvailability, ) from learning_resources.etl.constants import COMMON_HEADERS @@ -28,6 +30,8 @@ transform_levels, without_none, ) +from learning_resources.models import LearningResource +from learning_resources.serializers import LearningResourceInstructorSerializer from learning_resources.utils import get_year_and_semester from main.utils import clean_data, now_in_utc @@ -45,6 +49,7 @@ "platform", "offered_by", "etl_source", + "resource_type", ], ) OpenEdxExtractTransform = namedtuple( # noqa: PYI024 @@ -195,7 +200,7 @@ def _is_course_or_run_deleted(title): ) -def _filter_course(course): +def _filter_resource(config, resource): """ Filter courses to onces that are valid to ingest @@ -205,9 +210,12 @@ def _filter_course(course): Returns: bool: True if the course should be ingested """ - return not _is_course_or_run_deleted(course.get("title")) and course.get( - "course_runs", [] - ) + if config.resource_type == LearningResourceType.course.name: + return not _is_course_or_run_deleted(resource.get("title")) and resource.get( + "course_runs", [] + ) + else: + return True def _filter_course_run(course_run): @@ -223,7 +231,7 @@ def _filter_course_run(course_run): return not _is_course_or_run_deleted(course_run.get("title")) -def _transform_image(image_data: dict) -> dict: +def _transform_course_image(image_data: dict) -> dict: """Return the transformed image data if a url is provided""" if image_data and image_data.get("src"): return { @@ -233,6 +241,50 @@ def _transform_image(image_data: dict) -> dict: return None +def _transform_program_image(program_data) -> dict: + """Return the transformed image data if a url is provided""" + url = program_data.get("banner_image", {}).get("medium", {}).get("url") + if url: + return {"url": url, "description": program_data.get("title")} + return None + + +def _parse_course_dates(program, date_field): + """Return all the course run price values for the given date field""" + dates = [] + for course in program.get("courses", []): + if not course["excluded_from_search"]: + dates.extend( + [ + run[date_field] + for run in course["course_runs"] + if run["status"] == "published" and run[date_field] + ] + ) + return dates + + +def _add_course_prices(program): + """Sum all the course run price values""" + + def _get_course_price(course): + if not course["excluded_from_search"]: + for run in sorted( + course["course_runs"], key=lambda x: x["start"], reverse=False + ): + if run["status"] == "published": + return min( + [ + Decimal(seat["price"]) + for seat in run["seats"] + if seat["price"] != "0.00" + ] + ) + return Decimal(0.00) + + return sum(_get_course_price(course) for course in program.get("courses", [])) + + def _transform_course_run(config, course_run, course_last_modified, marketing_url): """ Transform a course run into the normalized data structure @@ -261,7 +313,7 @@ def _transform_course_run(config, course_run, course_last_modified, marketing_ur "published": _get_run_published(course_run), "enrollment_start": course_run.get("enrollment_start"), "enrollment_end": course_run.get("enrollment_end"), - "image": _transform_image(course_run.get("image")), + "image": _transform_course_image(course_run.get("image")), "availability": course_run.get("availability"), "url": marketing_url or "{}{}/course/".format(config.alt_url, course_run.get("key")), @@ -278,6 +330,80 @@ def _transform_course_run(config, course_run, course_last_modified, marketing_ur } +def _parse_program_instructors_topics(program): + """Get the instructors for each published course in a program""" + instructors = [] + topics = [] + course_ids = [course["key"] for course in program["courses"]] + courses = LearningResource.objects.filter( + readable_id__in=course_ids, + resource_type=LearningResourceType.course.name, + platform=PlatformType.edx.name, + published=True, + ) + for course in courses: + topics.extend([{"name": topic.name} for topic in course.topics.all()]) + run = ( + course.next_run + or course.runs.filter(published=True).order_by("-start_date").first() + ) + if run: + instructors.extend( + [ + LearningResourceInstructorSerializer(instance=instructor).data + for instructor in run.instructors.all() + ] + ) + return ( + sorted(instructors, key=lambda x: x["last_name"] or x["full_name"]), + sorted(topics, key=lambda x: x["name"]), + ) + + +def _transform_program_course(config, course): + return { + "readable_id": course.get("key"), + "etl_source": config.etl_source, + "platform": config.platform, + "resource_type": LearningResourceType.course.name, + "offered_by": {"code": config.offered_by}, + } + + +def _transform_program_run(program, program_last_modified, image): + """ + Transform a course run into the normalized data structure + + Args: + config (OpenEdxConfiguration): configuration for the openedx backend + + Returns: + dict: the tranformed course run data + """ + return { + "run_id": program.get("uuid"), + "title": program.get("title"), + "description": program.get("subtitle"), + "full_description": program.get("subtitle"), + "level": transform_levels([program.get("level_type_override")]), + "start_date": min(_parse_course_dates(program, "start"), default=None), + "end_date": max(_parse_course_dates(program, "end"), default=None), + "last_modified": program_last_modified, + "published": True, + "enrollment_start": min( + _parse_course_dates(program, "enrollment_start"), default=None + ), + "enrollment_end": max( + _parse_course_dates(program, "enrollment_end"), default=None + ), + "image": image, + "availability": RunAvailability.current.value, + "url": program.get("marketing_url"), + "prices": [_add_course_prices(program)], + "instructors": program.pop("instructors", []), + } + + def _transform_course(config, course): """ Filter courses to onces that are valid to ingest @@ -308,7 +434,7 @@ def _transform_course(config, course): "description": clean_data(course.get("short_description")), "full_description": clean_data(course.get("full_description")), "last_modified": last_modified, - "image": _transform_image(course.get("image")), + "image": _transform_course_image(course.get("image")), "url": marketing_url or "{}{}/course/".format(config.alt_url, course.get("key")), "topics": [ @@ -327,6 +453,69 @@ def _transform_course(config, course): } +def _transform_program(config, program): + """ + Filter courses to onces that are valid to ingest + + Args: + config (OpenEdxConfiguration): configuration for the openedx backend + program (dict): the course data + + Returns: + dict: the tranformed course data + """ + last_modified = _parse_openedx_datetime(program.get("data_modified_timestamp")) + marketing_url = program.get("marketing_url") + image = _transform_program_image(program) + instructors, topics = _parse_program_instructors_topics(program) + program["instructors"] = instructors + runs = [_transform_program_run(program, last_modified, image)] + has_certification = parse_certification(config.offered_by, runs) + return { + "readable_id": program.get("uuid"), + "etl_source": config.etl_source, + "platform": config.platform, + "resource_type": LearningResourceType.program.name, + "offered_by": {"code": config.offered_by}, + "title": program.get("title"), + "description": clean_data(program.get("subtitle")), + "full_description": clean_data(program.get("subtitle")), + "last_modified": last_modified, + "image": image, + "url": marketing_url + or "{}{}/course/".format(config.alt_url, program.get("key")), + "topics": topics, + "runs": runs, + "published": any(run["published"] is True for run in runs), + "certification": has_certification, + "certification_type": CertificationType.completion.name + if has_certification + else CertificationType.none.name, + "availability": Availability.anytime.name, + "courses": [ + _transform_program_course(config, course) + for course in program.get("courses", []) + ], + } + + +def _transform_resource(config, resource): + """ + Transform the extracted openedx data into our normalized data structure + + Args: + config (OpenEdxConfiguration): configuration for the openedx backend + resource (dict): the data for the resource + + Returns: + dict: the tranformed resource data + """ + if config.resource_type == LearningResourceType.course.name: + return _transform_course(config, resource) + else: + return _transform_program(config, resource) + + def openedx_extract_transform_factory(get_config): """ Factory for generating OpenEdx extract and transform functions based on the configuration @@ -375,10 +564,10 @@ def extract(api_datafile=None): url = config.api_url while url: - courses, url = _get_openedx_catalog_page(url, access_token) - yield from courses + resources, url = _get_openedx_catalog_page(url, access_token) + yield from resources - def transform(courses): + def transform(resources): """ Transforms the extracted openedx data into our normalized data structure @@ -392,9 +581,9 @@ def transform(courses): config = get_config() return [ - _transform_course(config, course) - for course in courses - if _filter_course(course) + _transform_resource(config, resource) + for resource in resources + if _filter_resource(config, resource) ] return OpenEdxExtractTransform( diff --git a/learning_resources/etl/openedx_test.py b/learning_resources/etl/openedx_test.py index cc5f1f0e13..a2f7666412 100644 --- a/learning_resources/etl/openedx_test.py +++ b/learning_resources/etl/openedx_test.py @@ -2,6 +2,7 @@ # pylint: disable=redefined-outer-name from datetime import datetime +from decimal import Decimal from urllib.parse import urlencode import pytest @@ -10,6 +11,8 @@ Availability, CertificationType, LearningResourceType, + OfferedBy, + PlatformType, RunAvailability, ) from learning_resources.etl.constants import COMMON_HEADERS, CourseNumberType @@ -17,15 +20,23 @@ OpenEdxConfiguration, openedx_extract_transform_factory, ) +from learning_resources.factories import ( + LearningResourceFactory, + LearningResourceOfferorFactory, + LearningResourcePlatformFactory, + LearningResourceRunFactory, +) +from learning_resources.serializers import LearningResourceInstructorSerializer from main.test_utils import any_instance_of +from main.utils import clean_data ACCESS_TOKEN = "invalid_access_token" # noqa: S105 @pytest.fixture -def openedx_config(): - """Fixture for the openedx config object""" - return OpenEdxConfiguration( +def openedx_common_config(): + """Fixture for the openedx common config object""" + return ( "fake-client-id", "fake-client-secret", "http://localhost/fake-access-token-url/", @@ -39,12 +50,38 @@ def openedx_config(): @pytest.fixture -def openedx_extract_transform(openedx_config): +def openedx_course_config(openedx_common_config): + """Fixture for the openedx config object""" + return OpenEdxConfiguration( + *openedx_common_config, + LearningResourceType.course.name, + ) + + +@pytest.fixture +def openedx_program_config(openedx_common_config): + """Fixture for the openedx config object""" + return OpenEdxConfiguration( + *openedx_common_config, + LearningResourceType.program.name, + ) + + +@pytest.fixture +def openedx_extract_transform_courses(openedx_course_config): """Fixture for generationg an extract/transform pair for the given config""" - return openedx_extract_transform_factory(lambda: openedx_config) + return openedx_extract_transform_factory(lambda: openedx_course_config) + +@pytest.fixture +def openedx_extract_transform_programs(openedx_program_config): + """Fixture for generationg an extract/transform pair for the given config""" + return openedx_extract_transform_factory(lambda: openedx_program_config) -def test_extract(mocked_responses, openedx_config, openedx_extract_transform): + +def test_extract( + mocked_responses, openedx_course_config, openedx_extract_transform_courses +): """Test the generated extract functoin walks the paginated results""" results1 = [1, 2, 3] results2 = [4, 5, 6] @@ -52,19 +89,19 @@ def test_extract(mocked_responses, openedx_config, openedx_extract_transform): mocked_responses.add( mocked_responses.POST, - openedx_config.access_token_url, + openedx_course_config.access_token_url, json={"access_token": ACCESS_TOKEN}, ) mocked_responses.add( mocked_responses.GET, - openedx_config.api_url, + openedx_course_config.api_url, json={"results": results1, "next": next_url}, ) mocked_responses.add( mocked_responses.GET, next_url, json={"results": results2, "next": None} ) - assert openedx_extract_transform.extract() == results1 + results2 + assert openedx_extract_transform_courses.extract() == results1 + results2 for call in mocked_responses.calls: # assert that headers contain our common ones @@ -73,8 +110,8 @@ def test_extract(mocked_responses, openedx_config, openedx_extract_transform): assert mocked_responses.calls[0].request.body == urlencode( { "grant_type": "client_credentials", - "client_id": openedx_config.client_id, - "client_secret": openedx_config.client_secret, + "client_id": openedx_course_config.client_id, + "client_secret": openedx_course_config.client_secret, "token_type": "jwt", } ) @@ -87,12 +124,12 @@ def test_extract(mocked_responses, openedx_config, openedx_extract_transform): @pytest.mark.usefixtures("mocked_responses") @pytest.mark.parametrize("config_arg_idx", range(6)) -def test_extract_disabled(openedx_config, config_arg_idx): +def test_extract_disabled(openedx_course_config, config_arg_idx): """ Verify that extract() exits with no API call if configuration is missing """ - args = list(openedx_config) + args = list(openedx_course_config) args[config_arg_idx] = None config = OpenEdxConfiguration(*args) @@ -123,8 +160,8 @@ def test_extract_disabled(openedx_config, config_arg_idx): ], ) def test_transform_course( # noqa: PLR0913 - openedx_config, - openedx_extract_transform, + openedx_course_config, + openedx_extract_transform_courses, mitx_course_data, has_runs, is_course_deleted, @@ -151,7 +188,7 @@ def test_transform_course( # noqa: PLR0913 if is_run_deleted: run["title"] = f"[delete] {run['title']}" - transformed_courses = openedx_extract_transform.transform(extracted) + transformed_courses = openedx_extract_transform_courses.transform(extracted) if is_course_deleted or not has_runs: assert transformed_courses == [] else: @@ -164,9 +201,9 @@ def test_transform_course( # noqa: PLR0913 "departments": ["15"], "description": "short_description", "full_description": "full description", - "platform": openedx_config.platform, - "etl_source": openedx_config.etl_source, - "offered_by": {"code": openedx_config.offered_by}, + "platform": openedx_course_config.platform, + "etl_source": openedx_course_config.etl_source, + "offered_by": {"code": openedx_course_config.offered_by}, "image": { "url": "https://prod-discovery.edx-cdn.org/media/course/image/ff1df27b-3c97-42ee-a9b3-e031ffd41a4f-747c9c2f216e.small.jpg", "description": "Image description", @@ -267,7 +304,7 @@ def test_transform_course( # noqa: PLR0913 @pytest.mark.parametrize("status", ["published", "other"]) @pytest.mark.parametrize("is_enrollable", [True, False]) def test_transform_course_availability_with_single_run( # noqa: PLR0913 - openedx_extract_transform, + openedx_extract_transform_courses, mitx_course_data, run_overrides, expected_availability, @@ -286,7 +323,7 @@ def test_transform_course_availability_with_single_run( # noqa: PLR0913 "status": status, } extracted[0]["course_runs"] = [run] - transformed_courses = openedx_extract_transform.transform([extracted[0]]) + transformed_courses = openedx_extract_transform_courses.transform([extracted[0]]) if status == "published" and is_enrollable: assert transformed_courses[0]["availability"] == expected_availability @@ -296,7 +333,7 @@ def test_transform_course_availability_with_single_run( # noqa: PLR0913 @pytest.mark.parametrize("has_dated", [True, False]) def test_transform_course_availability_with_multiple_runs( - openedx_extract_transform, mitx_course_data, has_dated + openedx_extract_transform_courses, mitx_course_data, has_dated ): """ Test that if course includes a single run corresponding to availability: "dated", @@ -329,9 +366,98 @@ def test_transform_course_availability_with_multiple_runs( if has_dated: runs.append(run2) extracted[0]["course_runs"] = runs - transformed_courses = openedx_extract_transform.transform([extracted[0]]) + transformed_courses = openedx_extract_transform_courses.transform([extracted[0]]) if has_dated: assert transformed_courses[0]["availability"] == Availability.dated.name else: assert transformed_courses[0]["availability"] is Availability.anytime.name + + +@pytest.mark.django_db +def test_transform_program( + openedx_program_config, + openedx_extract_transform_programs, + mitx_programs_data, +): # pylint: disable=too-many-arguments + """Test that the transform function normalizes and filters out data""" + platform = LearningResourcePlatformFactory.create(code=PlatformType.edx.name) + offeror = LearningResourceOfferorFactory.create(code=OfferedBy.mitx.name) + instructors = [] + topics = [] + for i in range(1, 4): + course = LearningResourceFactory.create( + readable_id=f"MITx+6.002.{i}x", + platform=platform, + offered_by=offeror, + is_course=True, + create_runs=False, + ) + topics.extend([topic.name for topic in course.topics.all()]) + LearningResourceRunFactory.create(learning_resource=course) + for run in course.runs.filter(published=True): + instructors.extend(run.instructors.all()) + extracted = mitx_programs_data + transformed_programs = openedx_extract_transform_programs.transform(extracted) + transformed_program = transformed_programs[0] + assert transformed_program == { + "title": extracted[0]["title"], + "readable_id": extracted[0]["uuid"], + "resource_type": LearningResourceType.program.name, + "description": clean_data(extracted[0]["subtitle"]), + "full_description": clean_data(extracted[0]["subtitle"]), + "platform": openedx_program_config.platform, + "etl_source": openedx_program_config.etl_source, + "offered_by": {"code": openedx_program_config.offered_by}, + "image": { + "url": extracted[0]["banner_image"]["medium"]["url"], + "description": extracted[0]["title"], + }, + "last_modified": any_instance_of(datetime), + "topics": [{"name": topic} for topic in sorted(topics)], + "url": extracted[0]["marketing_url"], + "published": True, + "certification": False, + "certification_type": CertificationType.none.name, + "availability": Availability.anytime.name, + "runs": ( + [ + { + "availability": RunAvailability.current.value, + "run_id": extracted[0]["uuid"], + "start_date": "2019-06-20T15:00:00Z", + "end_date": "2025-05-26T15:00:00Z", + "enrollment_end": "2025-05-17T15:00:00Z", + "enrollment_start": None, + "description": extracted[0]["subtitle"], + "full_description": extracted[0]["subtitle"], + "image": { + "url": extracted[0]["banner_image"]["medium"]["url"], + "description": extracted[0]["title"], + }, + "instructors": [ + LearningResourceInstructorSerializer(instructor).data + for instructor in sorted( + instructors, key=lambda x: x.last_name or x.full_name + ) + ], + "last_modified": any_instance_of(datetime), + "level": [], + "prices": [Decimal("567.00")], + "title": extracted[0]["title"], + "url": extracted[0]["marketing_url"], + "published": True, + } + ] + ), + "courses": [ + { + "etl_source": openedx_program_config.etl_source, + "offered_by": {"code": openedx_program_config.offered_by}, + "platform": openedx_program_config.platform, + "readable_id": f"MITx+6.002.{i}x", + "resource_type": LearningResourceType.course.name, + } + for i in range(1, 4) + ], + } diff --git a/learning_resources/etl/pipelines.py b/learning_resources/etl/pipelines.py index 793b520904..023263e58b 100644 --- a/learning_resources/etl/pipelines.py +++ b/learning_resources/etl/pipelines.py @@ -11,6 +11,7 @@ loaders, micromasters, mit_edx, + mit_edx_programs, mitxonline, ocw, oll, @@ -44,7 +45,7 @@ micromasters.extract, ) -mit_edx_etl = compose( +mit_edx_courses_etl = compose( load_courses( ETLSource.mit_edx.name, config=CourseLoaderConfig(prune=True), @@ -53,6 +54,17 @@ mit_edx.extract, ) +mit_edx_programs_etl = compose( + load_programs( + ETLSource.mit_edx.name, + config=ProgramLoaderConfig( + courses=CourseLoaderConfig(fetch_only=True), prune=True + ), + ), + mit_edx_programs.transform, + mit_edx_programs.extract, +) + mitxonline_programs_etl = compose( load_programs( ETLSource.mitxonline.name, diff --git a/learning_resources/etl/pipelines_test.py b/learning_resources/etl/pipelines_test.py index cda8889ad9..cc6411c46d 100644 --- a/learning_resources/etl/pipelines_test.py +++ b/learning_resources/etl/pipelines_test.py @@ -35,15 +35,15 @@ def reload_mocked_pipeline(*patchers): reload(pipelines) -def test_mit_edx_etl(): - """Verify that mit edx etl pipeline executes correctly""" +def test_mit_edx_courses_etl(): + """Verify that mit edx courses etl pipeline executes correctly""" with reload_mocked_pipeline( patch("learning_resources.etl.mit_edx.extract", autospec=True), patch("learning_resources.etl.mit_edx.transform", autospec=False), patch("learning_resources.etl.loaders.load_courses", autospec=True), ) as patches: mock_extract, mock_transform, mock_load_courses = patches - result = pipelines.mit_edx_etl() + result = pipelines.mit_edx_courses_etl() mock_extract.assert_called_once_with() @@ -60,6 +60,33 @@ def test_mit_edx_etl(): assert result == mock_load_courses.return_value +def test_mit_edx_programs_etl(): + """Verify that mit edx programs etl pipeline executes correctly""" + with reload_mocked_pipeline( + patch("learning_resources.etl.mit_edx_programs.extract", autospec=True), + patch("learning_resources.etl.mit_edx_programs.transform", autospec=False), + patch("learning_resources.etl.loaders.load_programs", autospec=True), + ) as patches: + mock_extract, mock_transform, mock_load_programs = patches + result = pipelines.mit_edx_programs_etl() + + mock_extract.assert_called_once_with() + + # each of these should be called with the return value of the extract + mock_transform.assert_called_once_with(mock_extract.return_value) + + # load_courses should be called *only* with the return value of transform + mock_load_programs.assert_called_once_with( + ETLSource.mit_edx.name, + mock_transform.return_value, + config=ProgramLoaderConfig( + courses=CourseLoaderConfig(fetch_only=True), prune=True + ), + ) + + assert result == mock_load_programs.return_value + + def test_mitxonline_programs_etl(): """Verify that mitxonline programs etl pipeline executes correctly""" with reload_mocked_pipeline( diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py index d17dbb4a67..848b3a0178 100644 --- a/learning_resources/tasks.py +++ b/learning_resources/tasks.py @@ -55,9 +55,10 @@ def get_mit_edx_data(api_datafile=None) -> int: api_datafile (str): If provided, use this file as the source of API data Otherwise, the API is queried directly. """ - courses = pipelines.mit_edx_etl(api_datafile) + courses = pipelines.mit_edx_courses_etl(api_datafile) + programs = pipelines.mit_edx_programs_etl(api_datafile) clear_search_cache() - return len(courses) + return len(courses + programs) @app.task diff --git a/learning_resources/tasks_test.py b/learning_resources/tasks_test.py index d765ff120e..3e15814c58 100644 --- a/learning_resources/tasks_test.py +++ b/learning_resources/tasks_test.py @@ -90,11 +90,12 @@ def test_get_micromasters_data(mocker): def test_get_mit_edx_data_valid(mocker): - """Verify that the get_mit_edx_data invokes the MIT edX ETL pipeline""" + """Verify that the get_mit_edx_data invokes the MIT edX ETL pipelines""" mock_pipelines = mocker.patch("learning_resources.tasks.pipelines") tasks.get_mit_edx_data.delay() - mock_pipelines.mit_edx_etl.assert_called_once_with(None) + mock_pipelines.mit_edx_courses_etl.assert_called_once_with(None) + mock_pipelines.mit_edx_programs_etl.assert_called_once_with(None) def test_get_mitxonline_data(mocker): diff --git a/main/settings_course_etl.py b/main/settings_course_etl.py index 3167e5037b..48a75dc93a 100644 --- a/main/settings_course_etl.py +++ b/main/settings_course_etl.py @@ -6,6 +6,7 @@ # EDX API Credentials EDX_API_URL = get_string("EDX_API_URL", None) +EDX_PROGRAMS_API_URL = get_string("EDX_PROGRAMS_API_URL", None) EDX_API_ACCESS_TOKEN_URL = get_string("EDX_API_ACCESS_TOKEN_URL", None) EDX_API_CLIENT_ID = get_string("EDX_API_CLIENT_ID", None) EDX_API_CLIENT_SECRET = get_string("EDX_API_CLIENT_SECRET", None) diff --git a/test_json/test_mitx_programs.json b/test_json/test_mitx_programs.json new file mode 100644 index 0000000000..4921e6d03f --- /dev/null +++ b/test_json/test_mitx_programs.json @@ -0,0 +1,1535 @@ +[ + { + "uuid": "927093e3-46ba-4f44-a861-0f8c7aec4f74", + "title": "Circuits and Electronics", + "subtitle": "Learn electronic circuit techniques and applications in designing microchips for smartphones, self-driving cars, computers, and the Internet.", + "type": "XSeries", + "type_attrs": { + "uuid": "a3d34232-2cca-4b35-8810-cef9bb1e0e77", + "slug": "xseries", + "coaching_supported": false + }, + "status": "active", + "marketing_slug": "mitx-circuits-and-electronics", + "marketing_url": "https://www.edx.org/xseries/mitx-circuits-and-electronics", + "banner_image": { + "large": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/927093e3-46ba-4f44-a861-0f8c7aec4f74-94e7a300643d.large.png", + "width": 1440, + "height": 480 + }, + "medium": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/927093e3-46ba-4f44-a861-0f8c7aec4f74-94e7a300643d.medium.png", + "width": 726, + "height": 242 + }, + "small": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/927093e3-46ba-4f44-a861-0f8c7a-ec4f74-94e7a300643d.small.png", + "width": 435, + "height": 145 + }, + "x-small": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/927093e3-46ba-4f44-a861-0f8c7aec4f74-94e7a300643d.x-small.png", + "width": 348, + "height": 116 + } + }, + "hidden": false, + "courses": [ + { + "key": "MITx+6.002.1x", + "uuid": "8f9c73ef-73f5-45dd-aca1-7dad3696f743", + "title": "Circuits and Electronics 1: Basic Circuit Analysis", + "course_runs": [ + { + "key": "course-v1:MITx+6.002.1x+2T2019", + "uuid": "4d39edd3-8c7a-4038-80cd-f6edab483b8a", + "title": "Circuits and Electronics 1: Basic Circuit Analysis", + "external_key": "", + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8f9c73ef-73f5-45dd-aca1-7dad3696f743-d417c0e28aed.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Learn techniques that are foundational to the design of microchips used in smartphones, self-driving cars, computers, and the Internet.

", + "marketing_url": "https://www.edx.org/course/circuits-and-electronics-1-basic-circuit-analysis?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "189.00", + "currency": "USD", + "upgrade_deadline": "2025-05-16T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "87611B2", + "bulk_sku": "DE4DB23" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "8C12550", + "bulk_sku": null + } + ], + "start": "2019-06-20T15:00:00Z", + "end": "2025-05-26T15:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": "2025-05-17T15:00:00Z", + "weeks_to_complete": 5, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "published", + "is_enrollable": true, + "is_marketable": true, + "availability": "Current", + "variant_id": null + } + ], + "entitlements": [ + { + "mode": "verified", + "price": "189.00", + "currency": "USD", + "sku": "CA912EE", + "expires": null + } + ], + "owners": [ + { + "uuid": "2a73d2ce-c34a-4e08-8223-83bca9d2f01d", + "key": "MITx", + "name": "Massachusetts Institute of Technology", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-0d040334059f.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-d4f180052205.png", + "organization_hex_color": null, + "data_modified_timestamp": "2024-07-08T11:50:26.966309Z" + } + ], + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8f9c73ef-73f5-45dd-aca1-7dad3696f743-d417c0e28aed.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Learn techniques that are foundational to the design of microchips used in smartphones, self-driving cars, computers, and the Internet.

", + "type": "e85a9f60-2da5-4413-96b7-dcb9b8c0e9f1", + "url_slug": null, + "course_type": "verified-audit", + "enterprise_subscription_inclusion": true, + "excluded_from_seo": false, + "excluded_from_search": false, + "course_run_statuses": ["published"] + }, + { + "key": "MITx+6.002.2x", + "uuid": "f6623bd8-ea35-42b2-880c-77a2f9c744b0", + "title": "Circuits and Electronics 2: Amplification, Speed, and Delay", + "course_runs": [ + { + "key": "course-v1:MITx+6.002.2x+2T2019", + "uuid": "289e51b0-04e7-4979-9ee7-9519af6a16f3", + "title": "Circuits and Electronics 2: Amplification, Speed, and Delay", + "external_key": "", + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/f6623bd8-ea35-42b2-880c-77a2f9c744b0-d8369d05cefc.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Learn how to speed up digital circuits and build amplifiers in the design of microchips used in smartphones, self-driving cars, computers, and the Internet.

", + "marketing_url": "https://www.edx.org/course/circuits-and-electronics-2-amplification-speed-and-delay?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "189.00", + "currency": "USD", + "upgrade_deadline": "2025-05-16T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "A1A0FA0", + "bulk_sku": "6ED8A2B" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "A6BBC32", + "bulk_sku": null + } + ], + "start": "2019-06-20T15:00:00Z", + "end": "2025-05-26T15:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": "2025-05-17T15:00:00Z", + "weeks_to_complete": 5, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "published", + "is_enrollable": true, + "is_marketable": true, + "availability": "Current", + "variant_id": null + } + ], + "entitlements": [ + { + "mode": "verified", + "price": "189.00", + "currency": "USD", + "sku": "9A03C7B", + "expires": null + } + ], + "owners": [ + { + "uuid": "2a73d2ce-c34a-4e08-8223-83bca9d2f01d", + "key": "MITx", + "name": "Massachusetts Institute of Technology", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-0d040334059f.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-d4f180052205.png", + "organization_hex_color": null, + "data_modified_timestamp": "2024-07-08T11:50:26.966309Z" + } + ], + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/f6623bd8-ea35-42b2-880c-77a2f9c744b0-d8369d05cefc.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Learn how to speed up digital circuits and build amplifiers in the design of microchips used in smartphones, self-driving cars, computers, and the Internet.

", + "type": "e85a9f60-2da5-4413-96b7-dcb9b8c0e9f1", + "url_slug": null, + "course_type": "verified-audit", + "enterprise_subscription_inclusion": true, + "excluded_from_seo": false, + "excluded_from_search": false, + "course_run_statuses": ["published"] + }, + { + "key": "MITx+6.002.3x", + "uuid": "aca3d8f7-8907-474c-9a62-58cca8c6254f", + "title": "Circuits and Electronics 3: Applications", + "course_runs": [ + { + "key": "course-v1:MITx+6.002.3x+2T2019", + "uuid": "5ac8faf4-1147-4ca6-b1ce-96aaf53cce30", + "title": "Circuits and Electronics 3: Applications", + "external_key": "", + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/aca3d8f7-8907-474c-9a62-58cca8c6254f-b9bf6f5f5ae8.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Learn about cool applications, op-amps and filters in the design of microchips used in smartphones, self-driving cars, computers, and the internet.

", + "marketing_url": "https://www.edx.org/course/circuits-and-electronics-3-applications?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "189.00", + "currency": "USD", + "upgrade_deadline": "2025-05-16T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "62ED93B", + "bulk_sku": "84F2929" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "319F510", + "bulk_sku": null + } + ], + "start": "2019-06-20T15:00:00Z", + "end": "2025-05-26T15:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": "2025-05-17T15:00:00Z", + "weeks_to_complete": 7, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "published", + "is_enrollable": true, + "is_marketable": true, + "availability": "Current", + "variant_id": null + } + ], + "entitlements": [ + { + "mode": "verified", + "price": "189.00", + "currency": "USD", + "sku": "7EF72A8", + "expires": null + } + ], + "owners": [ + { + "uuid": "2a73d2ce-c34a-4e08-8223-83bca9d2f01d", + "key": "MITx", + "name": "Massachusetts Institute of Technology", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-0d040334059f.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-d4f180052205.png", + "organization_hex_color": null, + "data_modified_timestamp": "2024-07-08T11:50:26.966309Z" + } + ], + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/aca3d8f7-8907-474c-9a62-58cca8c6254f-b9bf6f5f5ae8.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Learn about cool applications, op-amps and filters in the design of microchips used in smartphones, self-driving cars, computers, and the internet.

", + "type": "e85a9f60-2da5-4413-96b7-dcb9b8c0e9f1", + "url_slug": null, + "course_type": "verified-audit", + "enterprise_subscription_inclusion": true, + "excluded_from_seo": false, + "excluded_from_search": false, + "course_run_statuses": ["published"] + } + ], + "authoring_organizations": [ + { + "uuid": "2a73d2ce-c34a-4e08-8223-83bca9d2f01d", + "key": "MITx", + "name": "Massachusetts Institute of Technology", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-0d040334059f.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/2a73d2ce-c34a-4e08-8223-83bca9d2f01d-d4f180052205.png", + "organization_hex_color": null, + "data_modified_timestamp": "2024-07-08T11:50:26.966309Z" + } + ], + "card_image_url": "https://prod-discovery.edx-cdn.org/media/programs/card_images/927093e3-46ba-4f44-a861-0f8c7aec4f74-215df4ae5361.jpg", + "is_program_eligible_for_one_click_purchase": true, + "degree": null, + "curricula": [], + "marketing_hook": "Learn about electronic circuit techniques and applications\r", + "total_hours_of_effort": null, + "recent_enrollment_count": 20255, + "organization_short_code_override": "", + "organization_logo_override_url": null, + "primary_subject_override": null, + "level_type_override": null, + "language_override": null, + "labels": [], + "taxi_form": null, + "program_duration_override": null, + "data_modified_timestamp": "2024-08-28T07:14:23.507563Z", + "excluded_from_search": false, + "excluded_from_seo": false, + "has_ofac_restrictions": null, + "ofac_comment": "", + "course_run_statuses": ["published"], + "subscription_eligible": false, + "subscription_prices": [ + { + "price": "79.00", + "currency": "USD" + } + ] + }, + { + "uuid": "84811e48-94e6-4738-8e15-e019b289e374", + "title": "Accounting", + "subtitle": "Learn key concepts and skills in financial accounting, managerial accounting, and tax.", + "type": "MicroMasters", + "type_attrs": { + "uuid": "253dd13d-33a3-401b-bcaa-e0b0f0424777", + "slug": "micromasters", + "coaching_supported": false + }, + "status": "retired", + "marketing_slug": "iux-accounting", + "marketing_url": "https://www.edx.org/micromasters/iux-accounting", + "banner_image": { + "large": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/84811e48-94e6-4738-8e15-e019b289e374-c1e0d8478f47.large.png", + "width": 1440, + "height": 480 + }, + "medium": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/84811e48-94e6-4738-8e15-e019b289e374-c1e0d8478f47.medium.png", + "width": 726, + "height": 242 + }, + "small": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/84811e48-94e6-4738-8e15-e019b289e374-c1e0d8478f47.small.png", + "width": 435, + "height": 145 + }, + "x-small": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/84811e48-94e6-4738-8e15-e019b289e374-c1e0d8478f47.x-small.png", + "width": 348, + "height": 116 + } + }, + "hidden": true, + "courses": [ + { + "key": "IUx+BUKD-A500", + "uuid": "33b60d53-f5ac-4f44-8e87-c277289fb576", + "title": "Financial Reporting I", + "course_runs": [ + { + "key": "course-v1:IUx+BUKD-A500+2T2019", + "uuid": "333d356f-09c8-4897-894f-b1fa0b8e039b", + "title": "Financial Reporting I", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "marketing_url": "https://www.edx.org/course/financial-reporting-i?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "500.00", + "currency": "USD", + "upgrade_deadline": "2019-10-30T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "C20EA71", + "bulk_sku": "944FA6E" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "DA6810A", + "bulk_sku": null + } + ], + "start": "2019-08-20T05:00:00Z", + "end": "2019-11-09T23:59:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 12, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:IUx+BUKD-A500+3T2019", + "uuid": "5a6d79d0-a581-43f8-9e73-d8cbb6663a67", + "title": "Financial Reporting I", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "marketing_url": "https://www.edx.org/course/financial-reporting-i-3?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "500.00", + "currency": "USD", + "upgrade_deadline": "2020-02-05T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "5F72196", + "bulk_sku": "CC08D6C" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "4DA39A5", + "bulk_sku": null + } + ], + "start": "2019-11-12T05:00:00Z", + "end": "2020-02-15T05:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 12, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:IUx+BUKD-A500+2T2020", + "uuid": "80ed1973-1e0d-4df6-9079-a6d59818c523", + "title": "Financial Reporting I", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "marketing_url": "https://www.edx.org/course/financial-reporting-i-2?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "499.00", + "currency": "USD", + "upgrade_deadline": "2020-07-07T23:59:00Z", + "upgrade_deadline_override": "2020-07-07T23:59:00Z", + "credit_provider": null, + "credit_hours": null, + "sku": "3B035AD", + "bulk_sku": "8F96F45" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "5C78228", + "bulk_sku": null + } + ], + "start": "2020-05-19T05:00:00Z", + "end": "2020-08-06T05:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 12, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:IUx+BUKD-A500+3T2020", + "uuid": "e1ab8996-2d38-432b-b36f-b4c76bbe9cd7", + "title": "Financial Reporting I", + "external_key": "", + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "marketing_url": "https://www.edx.org/course/financial-reporting-i-course-v1iuxbukd-a5003t2020?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "468C99E", + "bulk_sku": null + }, + { + "type": "verified", + "price": "499.00", + "currency": "USD", + "upgrade_deadline": "2021-01-21T23:59:00Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "A677A5D", + "bulk_sku": "9E6F537" + } + ], + "start": "2020-11-17T05:00:00Z", + "end": "2021-02-19T20:01:00Z", + "go_live_date": "2020-02-26T05:00:00Z", + "enrollment_start": null, + "enrollment_end": "2020-12-15T00:00:00Z", + "weeks_to_complete": 12, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": false, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:IUx+BUKD-A500+2T2021", + "uuid": "e8c274c9-8c27-4fb5-a955-6e69e86c0d45", + "title": "Financial Reporting I", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "marketing_url": "https://www.edx.org/course/financial-reporting-i-course-v1iuxbukd-a5002t2021?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "60466B6", + "bulk_sku": null + }, + { + "type": "verified", + "price": "499.00", + "currency": "USD", + "upgrade_deadline": "2021-07-13T23:59:00Z", + "upgrade_deadline_override": "2021-07-13T23:59:00Z", + "credit_provider": null, + "credit_hours": null, + "sku": "C544705", + "bulk_sku": "738E9B4" + } + ], + "start": "2021-05-25T10:00:00Z", + "end": "2021-08-13T20:01:00Z", + "go_live_date": "2020-04-24T04:00:00Z", + "enrollment_start": null, + "enrollment_end": "2021-06-22T01:59:00Z", + "weeks_to_complete": 12, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": false, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:IUx+BUKD-A500+3T2021", + "uuid": "389ba5e9-e337-4a0d-adda-816fbf58efe3", + "title": "Financial Reporting I", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "marketing_url": "https://www.edx.org/course/financial-reporting-i-course-v1iuxbukd-a5003t2021?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "10C7C05", + "bulk_sku": null + }, + { + "type": "verified", + "price": "499.00", + "currency": "USD", + "upgrade_deadline": "2022-01-11T23:59:00Z", + "upgrade_deadline_override": "2022-01-11T23:59:00Z", + "credit_provider": null, + "credit_hours": null, + "sku": "6AB677F", + "bulk_sku": "94681A2" + } + ], + "start": "2021-11-16T11:00:00Z", + "end": "2022-02-18T04:59:00Z", + "go_live_date": "2020-11-20T05:00:00Z", + "enrollment_start": null, + "enrollment_end": "2021-12-14T23:59:00Z", + "weeks_to_complete": 12, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": false, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:IUx+BUKD-A500+2T2022", + "uuid": "b63c80d7-b533-4b47-be56-48bfd9899981", + "title": "Financial Reporting I", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "marketing_url": "https://www.edx.org/course/financial-reporting-i-course-v1iuxbukd-a5002t2022?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "3272D0A", + "bulk_sku": null + }, + { + "type": "verified", + "price": "499.00", + "currency": "USD", + "upgrade_deadline": "2022-06-08T03:59:00Z", + "upgrade_deadline_override": "2022-06-08T03:59:00Z", + "credit_provider": null, + "credit_hours": null, + "sku": "836E41E", + "bulk_sku": "C7CF0BE" + } + ], + "start": "2022-05-24T10:00:00Z", + "end": "2022-08-13T03:59:00Z", + "go_live_date": "2021-04-14T04:00:00Z", + "enrollment_start": null, + "enrollment_end": "2022-06-08T03:59:00Z", + "weeks_to_complete": 12, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": false, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + } + ], + "entitlements": [ + { + "mode": "verified", + "price": "499.00", + "currency": "USD", + "sku": "C88D5B7", + "expires": null + } + ], + "owners": [ + { + "uuid": "8946fb40-f288-4e05-9275-9e9b4689bda8", + "key": "IUx", + "name": "Indiana University", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/8946fb40-f288-4e05-9275-9e9b4689bda8-9c4c6df9d3bb.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/8946fb40-f288-4e05-9275-9e9b4689bda8-e58c545e2296.png", + "organization_hex_color": null, + "data_modified_timestamp": "2023-10-23T13:37:23.725096Z" + } + ], + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/33b60d53-f5ac-4f44-8e87-c277289fb576-94c15bb33734.small.jpeg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Earn a strong foundation in financial reporting concepts and methods, and use your skills to prepare and analyze financial statements.

", + "type": "e85a9f60-2da5-4413-96b7-dcb9b8c0e9f1", + "url_slug": null, + "course_type": "verified-audit", + "enterprise_subscription_inclusion": false, + "excluded_from_seo": false, + "excluded_from_search": false, + "course_run_statuses": ["archived"] + } + ], + "authoring_organizations": [ + { + "uuid": "8946fb40-f288-4e05-9275-9e9b4689bda8", + "key": "IUx", + "name": "Indiana University", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/8946fb40-f288-4e05-9275-9e9b4689bda8-9c4c6df9d3bb.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/8946fb40-f288-4e05-9275-9e9b4689bda8-e58c545e2296.png", + "organization_hex_color": null, + "data_modified_timestamp": "2023-10-23T13:37:23.725096Z" + } + ], + "card_image_url": "https://prod-discovery.edx-cdn.org/media/programs/card_images/84811e48-94e6-4738-8e15-e019b289e374-cdd9850ada79.jpg", + "is_program_eligible_for_one_click_purchase": false, + "degree": null, + "curricula": [], + "marketing_hook": "Gain skills in financial accounting, managerial accounting, and tax", + "total_hours_of_effort": null, + "recent_enrollment_count": -115, + "organization_short_code_override": "", + "organization_logo_override_url": null, + "primary_subject_override": null, + "level_type_override": null, + "language_override": null, + "labels": [], + "taxi_form": null, + "program_duration_override": null, + "data_modified_timestamp": "2024-08-28T07:13:53.011756Z", + "excluded_from_search": false, + "excluded_from_seo": false, + "has_ofac_restrictions": null, + "ofac_comment": "", + "course_run_statuses": ["archived"], + "subscription_eligible": null, + "subscription_prices": [] + }, + { + "uuid": "0445d855-1d24-480a-87ff-01a9186e237a", + "title": "Water Management", + "subtitle": "Explore water management concepts and technologies.", + "type": "XSeries", + "type_attrs": { + "uuid": "a3d34232-2cca-4b35-8810-cef9bb1e0e77", + "slug": "xseries", + "coaching_supported": false + }, + "status": "active", + "marketing_slug": "xseries/delft-university-of-technology-water-management", + "marketing_url": "https://www.edx.org/xseries/delft-university-of-technology-water-management", + "banner_image": { + "large": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/0445d855-1d24-480a-87ff-01a9186e237a.large.jpg", + "width": 1440, + "height": 480 + }, + "medium": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/0445d855-1d24-480a-87ff-01a9186e237a.medium.jpg", + "width": 726, + "height": 242 + }, + "small": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/0445d855-1d24-480a-87ff-01a9186e237a.small.jpg", + "width": 435, + "height": 145 + }, + "x-small": { + "url": "https://prod-discovery.edx-cdn.org/media/programs/banner_images/0445d855-1d24-480a-87ff-01a9186e237a.x-small.jpg", + "width": 348, + "height": 116 + } + }, + "hidden": false, + "courses": [ + { + "key": "Delftx+CTB3300WCx", + "uuid": "8d384724-c109-45d4-9a92-7920d3f74ef5", + "title": "Introduction to Water and Climate", + "course_runs": [ + { + "key": "DelftX/CTB3300WCx/2T2014", + "uuid": "895c2331-d427-4241-aad0-73ba4223f2c0", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "The basic elements of and the relation between water and climate are highlighted and further discussed together with their mutual coherence.", + "marketing_url": "https://www.edx.org/course/introduction-water-climate-delftx-ctb3300wcx?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": "2014-09-17T08:59:00Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "E1FB8DC", + "bulk_sku": null + }, + { + "type": "honor", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "5DED6EC", + "bulk_sku": null + }, + { + "type": "verified", + "price": "50.00", + "currency": "USD", + "upgrade_deadline": "2014-09-17T08:59:00Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "3EC8563", + "bulk_sku": null + } + ], + "start": "2014-08-26T10:00:00Z", + "end": "2014-11-04T11:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 9, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "1028f55f-04ea-4409-8542-6d2802ca4a5f", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+2015_T3", + "uuid": "a36f5673-6637-11e6-a8e3-22000bdde520", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "Explore how climate change, water availability, and engineering innovation are key challenges for our planet.", + "marketing_url": "https://www.edx.org/course/introduction-water-climate-delftx-ctb3300wcx-0?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "honor", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "1DB5A75", + "bulk_sku": null + }, + { + "type": "verified", + "price": "50.00", + "currency": "USD", + "upgrade_deadline": "2015-10-19T23:59:00Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "EFF47EC", + "bulk_sku": null + } + ], + "start": "2015-09-01T12:00:00Z", + "end": "2015-11-04T12:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 4, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "7833c89d-47f7-49bc-ba76-b38cbfa8f903", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+3T2016", + "uuid": "a36f5692-6637-11e6-a8e3-22000bdde520", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "Explore how climate change, water availability, and engineering innovation are key challenges for our planet.", + "marketing_url": "https://www.edx.org/course/introduction-water-climate-delftx-ctb3300wcx-1?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "5547B3C", + "bulk_sku": null + }, + { + "type": "verified", + "price": "50.00", + "currency": "USD", + "upgrade_deadline": "2016-11-28T23:58:00Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "C36751E", + "bulk_sku": null + } + ], + "start": "2016-09-06T12:00:00Z", + "end": "2016-11-15T12:00:00Z", + "go_live_date": null, + "enrollment_start": "2016-03-01T12:00:00Z", + "enrollment_end": null, + "weeks_to_complete": 7, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+3T2017", + "uuid": "ce65ed1e-53cf-4888-9ec6-a594cfbfc0a3", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.", + "marketing_url": "https://www.edx.org/course/introduction-water-climate-delftx-ctb3300wcx-2?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "8776217", + "bulk_sku": null + }, + { + "type": "verified", + "price": "50.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "EF51B77", + "bulk_sku": null + } + ], + "start": "2017-09-06T12:00:00Z", + "end": "2017-11-01T12:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 7, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+3T2018", + "uuid": "8ae0f3a9-3fdf-4153-9f06-0929de980bef", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.

", + "marketing_url": "https://www.edx.org/course/introduction-to-water-and-climate?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "50.00", + "currency": "USD", + "upgrade_deadline": "2018-10-20T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "51FACDC", + "bulk_sku": "3C3D06B" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "9CF1878", + "bulk_sku": null + } + ], + "start": "2018-09-05T12:00:00Z", + "end": "2018-10-30T12:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 7, + "pacing_type": "instructor_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+1T2019", + "uuid": "f2fcf0c4-ba9c-4765-8e90-3ad333110813", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.

", + "marketing_url": "https://www.edx.org/course/introduction-to-water-and-climate-0?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "50.00", + "currency": "USD", + "upgrade_deadline": "2019-05-22T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "26FDED2", + "bulk_sku": "B045F8E" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "669CB6C", + "bulk_sku": null + } + ], + "start": "2019-01-01T12:00:00Z", + "end": "2019-06-01T12:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 10, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+3T2019", + "uuid": "820730f5-24eb-41f9-960b-adf475bced0d", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.

", + "marketing_url": "https://www.edx.org/course/introduction-to-water-and-climate-2?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "verified", + "price": "50.00", + "currency": "USD", + "upgrade_deadline": "2021-01-04T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "E4CCAD3", + "bulk_sku": "5547ED9" + }, + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "59AB6FC", + "bulk_sku": null + } + ], + "start": "2019-09-01T00:00:00Z", + "end": "2021-01-14T00:00:00Z", + "go_live_date": null, + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 10, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+3T2021", + "uuid": "a7bed6f7-fddb-4979-97bf-544c48880eb6", + "title": "Introduction to Water and Climate", + "external_key": null, + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.

", + "marketing_url": "https://www.edx.org/course/introduction-to-water-and-climate-course-v1delftxctb3300wcx3t2021?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "4E77AD3", + "bulk_sku": null + }, + { + "type": "verified", + "price": "139.00", + "currency": "USD", + "upgrade_deadline": "2022-08-21T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "42C05D4", + "bulk_sku": "08A08C1" + } + ], + "start": "2021-09-01T10:00:00Z", + "end": "2022-08-31T10:00:00Z", + "go_live_date": "2021-04-27T22:00:00Z", + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 10, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+3T2022", + "uuid": "a3822c60-0e37-4bc9-8957-753068d2d662", + "title": "Introduction to Water and Climate", + "external_key": "", + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.

", + "marketing_url": "https://www.edx.org/course/introduction-to-water-and-climate-course-v1delftxctb3300wcx3t2022?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "E636C80", + "bulk_sku": null + }, + { + "type": "verified", + "price": "139.00", + "currency": "USD", + "upgrade_deadline": "2023-12-05T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "19897AC", + "bulk_sku": "FCB1B8F" + } + ], + "start": "2022-09-01T10:00:00Z", + "end": "2023-12-15T10:00:00Z", + "go_live_date": "2022-04-30T22:00:00Z", + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 10, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "unpublished", + "is_enrollable": true, + "is_marketable": false, + "availability": "Archived", + "variant_id": null + }, + { + "key": "course-v1:Delftx+CTB3300WCx+1T2024", + "uuid": "b127d07e-3b2d-4bdb-970a-44b871ad6237", + "title": "Introduction to Water and Climate", + "external_key": "", + "fixed_price_usd": null, + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.

", + "marketing_url": "https://www.edx.org/course/introduction-to-water-and-climate-course-v1-delftx-ctb3300wcx-1t2024?utm_source=ocwprod-mit-opencourseware&utm_medium=affiliate_partner", + "seats": [ + { + "type": "audit", + "price": "0.00", + "currency": "USD", + "upgrade_deadline": null, + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "007073A", + "bulk_sku": null + }, + { + "type": "verified", + "price": "139.00", + "currency": "USD", + "upgrade_deadline": "2025-05-18T23:59:59Z", + "upgrade_deadline_override": null, + "credit_provider": null, + "credit_hours": null, + "sku": "11EC6DB", + "bulk_sku": "636D2D9" + } + ], + "start": "2024-02-28T11:00:00Z", + "end": "2025-05-28T10:00:00Z", + "go_live_date": "2024-02-08T23:00:00Z", + "enrollment_start": null, + "enrollment_end": null, + "weeks_to_complete": 10, + "pacing_type": "self_paced", + "type": "verified", + "restriction_type": null, + "run_type": "df9c20c1-9b54-40a5-bae3-7fda48d84141", + "status": "published", + "is_enrollable": true, + "is_marketable": true, + "availability": "Current", + "variant_id": null + } + ], + "entitlements": [ + { + "mode": "verified", + "price": "139.00", + "currency": "USD", + "sku": "7ABFFB0", + "expires": null + } + ], + "owners": [ + { + "uuid": "c484a523-d396-4aff-90f4-bb7e82e16bf6", + "key": "DelftX", + "name": "Delft University of Technology", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/c484a523-d396-4aff-90f4-bb7e82e16bf6-c43b6cb39bcc.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/c484a523-d396-4aff-90f4-bb7e82e16bf6-f9e6cc4a4c94.png", + "organization_hex_color": null, + "data_modified_timestamp": "2024-02-01T15:45:47.314868Z" + } + ], + "image": { + "src": "https://prod-discovery.edx-cdn.org/media/course/image/8d384724-c109-45d4-9a92-7920d3f74ef5-4e859aa4bd33.small.jpg", + "description": null, + "height": null, + "width": null + }, + "short_description": "

Water is a crucial element in climate and for society. Find out about the latest engineering interventions for water management in rivers, coasts and the urban environment.

", + "type": "e85a9f60-2da5-4413-96b7-dcb9b8c0e9f1", + "url_slug": null, + "course_type": "verified-audit", + "enterprise_subscription_inclusion": true, + "excluded_from_seo": false, + "excluded_from_search": false, + "course_run_statuses": ["archived", "published"] + } + ], + "authoring_organizations": [ + { + "uuid": "c484a523-d396-4aff-90f4-bb7e82e16bf6", + "key": "DelftX", + "name": "Delft University of Technology", + "auto_generate_course_run_keys": true, + "certificate_logo_image_url": "https://prod-discovery.edx-cdn.org/organization/certificate_logos/c484a523-d396-4aff-90f4-bb7e82e16bf6-c43b6cb39bcc.png", + "logo_image_url": "https://prod-discovery.edx-cdn.org/organization/logos/c484a523-d396-4aff-90f4-bb7e82e16bf6-f9e6cc4a4c94.png", + "organization_hex_color": null, + "data_modified_timestamp": "2024-02-01T15:45:47.314868Z" + } + ], + "card_image_url": "https://prod-discovery.edx-cdn.org/media/programs/card_images/0445d855-1d24-480a-87ff-01a9186e237a-c1d6687579f0.jpg", + "is_program_eligible_for_one_click_purchase": false, + "degree": null, + "curricula": [], + "marketing_hook": "Explore water management concepts and technologies", + "total_hours_of_effort": null, + "recent_enrollment_count": 3462, + "organization_short_code_override": "", + "organization_logo_override_url": null, + "primary_subject_override": null, + "level_type_override": null, + "language_override": null, + "labels": [], + "taxi_form": null, + "program_duration_override": null, + "data_modified_timestamp": "2024-08-28T07:12:51.701135Z", + "excluded_from_search": false, + "excluded_from_seo": false, + "has_ofac_restrictions": null, + "ofac_comment": "", + "course_run_statuses": ["archived", "published"], + "subscription_eligible": false, + "subscription_prices": [ + { + "price": "79.00", + "currency": "USD" + } + ] + } +] From de97e2f140a9439ad758e5598ed905a12369f505 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Tue, 10 Sep 2024 09:40:28 -0400 Subject: [PATCH 2/3] Just in case, filter out "deleted" programs too --- learning_resources/etl/openedx.py | 2 +- learning_resources/etl/openedx_test.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/learning_resources/etl/openedx.py b/learning_resources/etl/openedx.py index ec1e83c7ef..47857d1dad 100644 --- a/learning_resources/etl/openedx.py +++ b/learning_resources/etl/openedx.py @@ -215,7 +215,7 @@ def _filter_resource(config, resource): "course_runs", [] ) else: - return True + return not _is_course_or_run_deleted(resource.get("title")) def _filter_course_run(course_run): diff --git a/learning_resources/etl/openedx_test.py b/learning_resources/etl/openedx_test.py index a2f7666412..048e17cf49 100644 --- a/learning_resources/etl/openedx_test.py +++ b/learning_resources/etl/openedx_test.py @@ -18,6 +18,7 @@ from learning_resources.etl.constants import COMMON_HEADERS, CourseNumberType from learning_resources.etl.openedx import ( OpenEdxConfiguration, + _filter_resource, openedx_extract_transform_factory, ) from learning_resources.factories import ( @@ -461,3 +462,14 @@ def test_transform_program( for i in range(1, 4) ], } + + +@pytest.mark.parametrize("deleted", [True, False]) +def test_filter_resource(openedx_course_config, openedx_program_config, deleted): + """Test that the filter_resource function filters out resources with DELETE in the title""" + resource = { + "title": "delete" if deleted else "Valid title", + "course_runs": [{"run_id": "id1"}], + } + assert _filter_resource(openedx_course_config, resource) is not deleted + assert _filter_resource(openedx_program_config, resource) is not deleted From e504a81f36a1ffb1cd5e80640ca9669856c2e685 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Tue, 10 Sep 2024 14:46:50 -0400 Subject: [PATCH 3/3] feedback --- learning_resources/etl/mit_edx_programs.py | 13 ++-- learning_resources/etl/openedx.py | 89 ++++++++++++++-------- learning_resources/tasks.py | 8 +- 3 files changed, 67 insertions(+), 43 deletions(-) diff --git a/learning_resources/etl/mit_edx_programs.py b/learning_resources/etl/mit_edx_programs.py index 006093f548..5abc881893 100644 --- a/learning_resources/etl/mit_edx_programs.py +++ b/learning_resources/etl/mit_edx_programs.py @@ -16,15 +16,15 @@ log = logging.getLogger() -def _is_mit_program(program): +def _is_mit_program(program: dict) -> bool: """ - Helper function to determine if a course is an MIT course + Helper function to determine if a program is an MIT program Args: - course (dict): The JSON object representing the course with all its course runs + program (dict): The JSON object representing the program with all its courses Returns: - bool: indicates whether the course is owned by MIT + bool: indicates whether the program is owned by MIT """ # noqa: D401 return ( any( @@ -38,7 +38,7 @@ def _is_mit_program(program): def get_open_edx_config(): """ - Return the OpenEdxConfiguration for edX. + Return the program OpenEdxConfiguration for edX. """ required_settings = [ "EDX_API_CLIENT_ID", @@ -68,5 +68,6 @@ def get_open_edx_config(): # use the OpenEdx factory to create our extract and transform funcs extract, _transform = openedx_extract_transform_factory(get_open_edx_config) -# modified transform function that filters the course list to ones that pass the _is_mit_course() predicate # noqa: E501 +# modified transform function that filters the program list to ones +# that pass the _is_mit_program() predicate transform = compose(_transform, curried.filter(_is_mit_program)) diff --git a/learning_resources/etl/openedx.py b/learning_resources/etl/openedx.py index 47857d1dad..fe01dd9678 100644 --- a/learning_resources/etl/openedx.py +++ b/learning_resources/etl/openedx.py @@ -178,17 +178,17 @@ def _get_course_availability(course): return None -def _is_course_or_run_deleted(title): +def _is_resource_or_run_deleted(title: str) -> bool: """ Returns True if '[delete]', 'delete ' (note the ending space character) - exists in a course's title or if the course title equals 'delete' for the - purpose of skipping the course + exists in a resource's title or if the resource title equals 'delete' for the + purpose of skipping the resource/run Args: - title (str): The course.title of the course + title (str): The title of the resource Returns: - bool: True if the course or run should be considered deleted + bool: True if the resource or run should be considered deleted """ # noqa: D401 title = title.strip().lower() @@ -202,7 +202,7 @@ def _is_course_or_run_deleted(title): def _filter_resource(config, resource): """ - Filter courses to onces that are valid to ingest + Filter resources to onces that are valid to ingest Args: course (dict): the course data @@ -211,24 +211,24 @@ def _filter_resource(config, resource): bool: True if the course should be ingested """ if config.resource_type == LearningResourceType.course.name: - return not _is_course_or_run_deleted(resource.get("title")) and resource.get( + return not _is_resource_or_run_deleted(resource.get("title")) and resource.get( "course_runs", [] ) else: - return not _is_course_or_run_deleted(resource.get("title")) + return not _is_resource_or_run_deleted(resource.get("title")) -def _filter_course_run(course_run): +def _filter_resource_run(resource_run): """ - Filter course runs to onces that are valid to ingest + Filter resource runs to ones that are valid to ingest Args: - course_run (dict): the course run data + resource_run (dict): the resource run data Returns: - bool: True if the course run should be ingested + bool: True if the resource run should be ingested """ - return not _is_course_or_run_deleted(course_run.get("title")) + return not _is_resource_or_run_deleted(resource_run.get("title")) def _transform_course_image(image_data: dict) -> dict: @@ -264,8 +264,16 @@ def _parse_course_dates(program, date_field): return dates -def _add_course_prices(program): - """Sum all the course run price values""" +def _sum_course_prices(program: dict) -> Decimal: + """ + Sum all the course run price values for the program + + Args: + program (dict): the program data + + Returns: + Decimal: the sum of all the course prices + """ def _get_course_price(course): if not course["excluded_from_search"]: @@ -360,7 +368,18 @@ def _parse_program_instructors_topics(program): ) -def _transform_program_course(config, course): +def _transform_program_course(config: OpenEdxConfiguration, course: dict) -> dict: + """ + Transform a program's course dict to a normalized data structure + + Args: + config (OpenEdxConfiguration): configuration for the openedx backend + course (dict): the course data + + Returns: + dict: the transformed course data for the program + + """ return { "readable_id": course.get("key"), "etl_source": config.etl_source, @@ -370,15 +389,19 @@ def _transform_program_course(config, course): } -def _transform_program_run(program, program_last_modified, image): +def _transform_program_run( + program: dict, program_last_modified: str, image: dict +) -> dict: """ - Transform a course run into the normalized data structure + Transform program data into the normalized data structure for its run Args: - config (OpenEdxConfiguration): configuration for the openedx backend + program (dict): the program data + program_last_modified (str): the last modified date string for the program + image (dict): the image data for the program Returns: - dict: the tranformed course run data + dict: the transformed program run data """ return { "run_id": program.get("uuid"), @@ -399,12 +422,12 @@ def _transform_program_run(program, program_last_modified, image): "image": image, "availability": RunAvailability.current.value, "url": program.get("marketing_url"), - "prices": [_add_course_prices(program)], + "prices": [_sum_course_prices(program)], "instructors": program.pop("instructors", []), } -def _transform_course(config, course): +def _transform_course(config: OpenEdxConfiguration, course: dict) -> dict: """ Filter courses to onces that are valid to ingest @@ -420,7 +443,7 @@ def _transform_course(config, course): runs = [ _transform_course_run(config, course_run, last_modified, marketing_url) for course_run in course.get("course_runs", []) - if _filter_course_run(course_run) + if _filter_resource_run(course_run) ] has_certification = parse_certification(config.offered_by, runs) return { @@ -453,16 +476,16 @@ def _transform_course(config, course): } -def _transform_program(config, program): +def _transform_program(config: OpenEdxConfiguration, program: dict) -> dict: """ - Filter courses to onces that are valid to ingest + Transform raw program data into a normalized data structure Args: config (OpenEdxConfiguration): configuration for the openedx backend - program (dict): the course data + program (dict): the program data Returns: - dict: the tranformed course data + dict: the tranformed program data """ last_modified = _parse_openedx_datetime(program.get("data_modified_timestamp")) marketing_url = program.get("marketing_url") @@ -499,9 +522,9 @@ def _transform_program(config, program): } -def _transform_resource(config, resource): +def _transform_resource(config: OpenEdxConfiguration, resource: dict) -> dict: """ - Transform the extracted openedx data into our normalized data structure + Transform the extracted openedx resource data into our normalized data structure Args: config (OpenEdxConfiguration): configuration for the openedx backend @@ -516,7 +539,7 @@ def _transform_resource(config, resource): return _transform_program(config, resource) -def openedx_extract_transform_factory(get_config): +def openedx_extract_transform_factory(get_config: callable) -> OpenEdxExtractTransform: """ Factory for generating OpenEdx extract and transform functions based on the configuration @@ -567,15 +590,15 @@ def extract(api_datafile=None): resources, url = _get_openedx_catalog_page(url, access_token) yield from resources - def transform(resources): + def transform(resources: list[dict]) -> list[dict]: """ Transforms the extracted openedx data into our normalized data structure Args: - list of dict: the merged catalog responses + list of dict: the merged resources responses Returns: - list of dict: the tranformed courses data + list of dict: the tranformed resources data """ # noqa: D401 config = get_config() diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py index 848b3a0178..b36f70487c 100644 --- a/learning_resources/tasks.py +++ b/learning_resources/tasks.py @@ -58,7 +58,7 @@ def get_mit_edx_data(api_datafile=None) -> int: courses = pipelines.mit_edx_courses_etl(api_datafile) programs = pipelines.mit_edx_programs_etl(api_datafile) clear_search_cache() - return len(courses + programs) + return len(courses) + len(programs) @app.task @@ -67,7 +67,7 @@ def get_mitxonline_data() -> int: courses = pipelines.mitxonline_courses_etl() programs = pipelines.mitxonline_programs_etl() clear_search_cache() - return len(courses + programs) + return len(courses) + len(programs) @app.task @@ -90,7 +90,7 @@ def get_prolearn_data(): courses = pipelines.prolearn_courses_etl() programs = pipelines.prolearn_programs_etl() clear_search_cache() - return len(programs + courses) + return len(courses) + len(programs) @app.task @@ -106,7 +106,7 @@ def get_xpro_data(): courses = pipelines.xpro_courses_etl() programs = pipelines.xpro_programs_etl() clear_search_cache() - return len(courses + programs) + return len(courses) + len(programs) @app.task