From 8359a68ed1698e5a6230956d000ae173e6dfab84 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Tue, 17 Sep 2024 09:29:27 -0400 Subject: [PATCH] Delivery and availability fields for runs --- frontends/api/src/generated/v1/api.ts | 20 ++++++ .../test-utils/factories/learningResources.ts | 9 +++ learning_resources/constants.py | 6 +- learning_resources/etl/constants.py | 10 +++ learning_resources/etl/deduplication.py | 42 ------------- learning_resources/etl/deduplication_test.py | 61 ------------------- learning_resources/etl/loaders.py | 35 +++++++---- learning_resources/etl/loaders_test.py | 39 +++++++----- learning_resources/etl/micromasters.py | 1 + learning_resources/etl/micromasters_test.py | 1 + learning_resources/etl/mitxonline.py | 12 ++-- learning_resources/etl/mitxonline_test.py | 17 +++--- learning_resources/etl/ocw.py | 14 ++--- learning_resources/etl/ocw_test.py | 9 ++- learning_resources/etl/oll.py | 9 +-- learning_resources/etl/oll_test.py | 6 +- learning_resources/etl/openedx.py | 46 ++++++++++++-- learning_resources/etl/openedx_test.py | 20 +++--- learning_resources/etl/prolearn.py | 3 + learning_resources/etl/prolearn_test.py | 4 ++ learning_resources/etl/sloan.py | 25 ++++---- learning_resources/etl/sloan_test.py | 32 +++++----- learning_resources/etl/utils.py | 14 ++--- learning_resources/etl/utils_test.py | 40 ++++++------ learning_resources/etl/xpro.py | 19 +++--- learning_resources/etl/xpro_test.py | 6 ++ learning_resources/factories.py | 9 +-- .../0046_learningresource_certification.py | 4 +- ...2_learningresource_delivery_format_pace.py | 2 +- .../0064_learningresourcerun_delivery.py | 55 +++++++++++++++++ .../0065_learningresourcerun_availability.py | 36 +++++++++++ learning_resources/models.py | 23 +++++-- learning_resources/serializers.py | 5 +- learning_resources/tasks.py | 4 +- learning_resources/tasks_test.py | 2 +- learning_resources_search/constants.py | 7 +++ openapi/specs/v1.yaml | 28 +++++++++ test_json/test_sloan_courses.json | 12 ---- 38 files changed, 417 insertions(+), 270 deletions(-) delete mode 100644 learning_resources/etl/deduplication.py delete mode 100644 learning_resources/etl/deduplication_test.py create mode 100644 learning_resources/migrations/0064_learningresourcerun_delivery.py create mode 100644 learning_resources/migrations/0065_learningresourcerun_availability.py diff --git a/frontends/api/src/generated/v1/api.ts b/frontends/api/src/generated/v1/api.ts index baae59bae7..7d9fe5fc12 100644 --- a/frontends/api/src/generated/v1/api.ts +++ b/frontends/api/src/generated/v1/api.ts @@ -2170,6 +2170,12 @@ export interface LearningResourceRun { * @memberof LearningResourceRun */ level: Array + /** + * + * @type {Array} + * @memberof LearningResourceRun + */ + delivery: Array /** * * @type {string} @@ -2272,7 +2278,14 @@ export interface LearningResourceRun { * @memberof LearningResourceRun */ checksum?: string | null + /** + * + * @type {AvailabilityEnum} + * @memberof LearningResourceRun + */ + availability?: AvailabilityEnum | null } + /** * * @export @@ -2420,7 +2433,14 @@ export interface LearningResourceRunRequest { * @memberof LearningResourceRunRequest */ checksum?: string | null + /** + * + * @type {AvailabilityEnum} + * @memberof LearningResourceRunRequest + */ + availability?: AvailabilityEnum | null } + /** * Serializer for LearningResourceSchool model, including list of departments * @export diff --git a/frontends/api/src/test-utils/factories/learningResources.ts b/frontends/api/src/test-utils/factories/learningResources.ts index 7de5efb2e0..8c11ddf195 100644 --- a/frontends/api/src/test-utils/factories/learningResources.ts +++ b/frontends/api/src/test-utils/factories/learningResources.ts @@ -28,6 +28,8 @@ import type { VideoResource, } from "api" import { + AvailabilityEnum, + DeliveryEnum, ResourceTypeEnum, LearningResourceRunLevelInnerCodeEnum, PlatformEnum, @@ -177,6 +179,13 @@ const learningResourceRun: Factory = (overrides = {}) => { languages: maybe(() => repeat(language, { min: 0, max: 3 })), start_date: start.toISOString(), end_date: end.toISOString(), + availability: faker.helpers.arrayElement(Object.values(AvailabilityEnum)), + delivery: [ + { + code: faker.helpers.arrayElement(Object.values(DeliveryEnum)), + name: uniqueEnforcerWords.enforce(() => faker.lorem.words()), + }, + ], level: [ { code: faker.helpers.arrayElement( diff --git a/learning_resources/constants.py b/learning_resources/constants.py index 71c464c32a..bdc28960b7 100644 --- a/learning_resources/constants.py +++ b/learning_resources/constants.py @@ -6,9 +6,9 @@ FAVORITES_TITLE = "Favorites" -class RunAvailability(ExtendedEnum): +class RunStatus(ExtendedEnum): """ - Enum for Course availability options dictated by edX API values. + Enum for run status options dictated by edX API values. """ current = "Current" @@ -270,7 +270,7 @@ class LevelType(ExtendedEnum): class LearningResourceFormat(ExtendedEnum): - """Enum for resource learning_format""" + """Enum for resource learning format""" online = "Online" hybrid = "Hybrid" diff --git a/learning_resources/etl/constants.py b/learning_resources/etl/constants.py index 4c5d92beab..5942e5c46c 100644 --- a/learning_resources/etl/constants.py +++ b/learning_resources/etl/constants.py @@ -1,6 +1,9 @@ """Constants for learning_resources ETL processes""" from collections import namedtuple +from dataclasses import dataclass, field +from datetime import datetime +from decimal import Decimal from enum import Enum from django.conf import settings @@ -106,3 +109,10 @@ class ContentTagCategory(ExtendedEnum): "Assignments": ContentTagCategory.problem_sets.value, # Can add more here if ever needed, in format tag_name:category } + + +@dataclass +class ResourceNextRunConfig: + next_start_date: datetime = None + prices: list[Decimal] = field(default_factory=list) + availability: str = None diff --git a/learning_resources/etl/deduplication.py b/learning_resources/etl/deduplication.py deleted file mode 100644 index 9e69e1e860..0000000000 --- a/learning_resources/etl/deduplication.py +++ /dev/null @@ -1,42 +0,0 @@ -"""Functions to combine duplicate courses""" - -from learning_resources.constants import RunAvailability - - -def get_most_relevant_run(runs): - """ - Helper function to determine the most relevant course run. - - Args: - runs (QuerySet): a set of LearningResourseRun objects - Returns: - A LearningResourseRun object - """ # noqa: D401 - - # if there is a current run in the set pick it - most_relevant_run = next( - (run for run in runs if run.availability == RunAvailability.current.value), - None, - ) - - if not most_relevant_run: - # if there a future runs in the set, pick the one with earliest start date - runs = runs.order_by("start_date") - most_relevant_run = next( - ( - run - for run in runs - if run.availability - in [ - RunAvailability.upcoming.value, - RunAvailability.starting_soon.value, - ] - ), - None, - ) - - if not most_relevant_run: - # get latest past run by start date - most_relevant_run = next(run for run in runs.reverse()) - - return most_relevant_run diff --git a/learning_resources/etl/deduplication_test.py b/learning_resources/etl/deduplication_test.py deleted file mode 100644 index 08aeb47b2f..0000000000 --- a/learning_resources/etl/deduplication_test.py +++ /dev/null @@ -1,61 +0,0 @@ -"""Tests for the deduplication ETL functions""" - -from datetime import UTC, datetime - -import pytest - -from learning_resources.constants import RunAvailability -from learning_resources.etl.deduplication import get_most_relevant_run -from learning_resources.factories import LearningResourceRunFactory -from learning_resources.models import LearningResourceRun - - -@pytest.mark.django_db -def test_get_most_relevant_run(): - """Verify that most_relevant_run returns the correct run""" - - most_relevant_run = LearningResourceRunFactory.create( - availability=RunAvailability.archived.value, - start_date=datetime(2019, 10, 1, tzinfo=UTC), - run_id="1", - ) - LearningResourceRunFactory.create( - availability=RunAvailability.archived.value, - start_date=datetime(2018, 10, 1, tzinfo=UTC), - run_id="2", - ) - - assert ( - get_most_relevant_run(LearningResourceRun.objects.filter(run_id__in=["1", "2"])) - == most_relevant_run - ) - - most_relevant_run = LearningResourceRunFactory.create( - availability=RunAvailability.upcoming.value, - start_date=datetime(2017, 10, 1, tzinfo=UTC), - run_id="3", - ) - - LearningResourceRunFactory.create( - availability=RunAvailability.upcoming.value, - start_date=datetime(2020, 10, 1, tzinfo=UTC), - run_id="4", - ) - - assert ( - get_most_relevant_run( - LearningResourceRun.objects.filter(run_id__in=["1", "2", "3", "4"]) - ) - == most_relevant_run - ) - - most_relevant_run = LearningResourceRunFactory.create( - availability=RunAvailability.current.value, run_id="5" - ) - - assert ( - get_most_relevant_run( - LearningResourceRun.objects.filter(run_id__in=["1", "2", "3", "4", "5"]) - ) - == most_relevant_run - ) diff --git a/learning_resources/etl/loaders.py b/learning_resources/etl/loaders.py index 3e058bcff8..c360535aa1 100644 --- a/learning_resources/etl/loaders.py +++ b/learning_resources/etl/loaders.py @@ -1,8 +1,6 @@ """learning_resources data loaders""" -import datetime import logging -from decimal import Decimal from django.contrib.auth import get_user_model from django.db import transaction @@ -12,7 +10,7 @@ LearningResourceRelationTypes, LearningResourceType, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import ( CONTENT_TAG_CATEGORIES, @@ -20,6 +18,7 @@ ContentTagCategory, CourseLoaderConfig, ProgramLoaderConfig, + ResourceNextRunConfig, ) from learning_resources.etl.exceptions import ExtractException from learning_resources.etl.utils import most_common_topics @@ -118,9 +117,18 @@ def load_departments( return resource.departments.all() -def load_next_start_date_and_prices( +def load_run_dependent_values( resource: LearningResource, -) -> tuple[datetime.time | None, list[Decimal]]: +) -> ResourceNextRunConfig: + """ + Assign prices, availability, and next_start_date to a resource based on its runs + + Args: + resource (LearningResource): the resource to update + + Returns: + tuple[datetime.time | None, list[Decimal], str]: date, prices, and availability + """ next_upcoming_run = resource.next_run if next_upcoming_run: resource.next_start_date = next_upcoming_run.start_date @@ -130,13 +138,18 @@ def load_next_start_date_and_prices( next_upcoming_run or resource.runs.filter(published=True).order_by("-start_date").first() ) + resource.availability = best_run.availability if best_run else resource.availability resource.prices = ( best_run.prices if resource.certification and best_run and best_run.prices else [] ) resource.save() - return resource.next_start_date, resource.prices + return ResourceNextRunConfig( + next_start_date=resource.next_start_date, + prices=resource.prices, + availability=resource.availability, + ) def load_instructors( @@ -234,12 +247,10 @@ def load_run( """ run_id = run_data.pop("run_id") image_data = run_data.pop("image", None) + status = run_data.pop("status", None) instructors_data = run_data.pop("instructors", []) - if ( - run_data.get("availability") == RunAvailability.archived.value - or learning_resource.certification is False - ): + if status == RunStatus.archived.value or learning_resource.certification is False: # Archived runs or runs of resources w/out certificates should not have prices run_data["prices"] = [] else: @@ -430,7 +441,7 @@ def load_course( run.save() resource_run_unpublished_actions(run) - load_next_start_date_and_prices(learning_resource) + load_run_dependent_values(learning_resource) load_topics(learning_resource, topics_data) load_offered_by(learning_resource, offered_bys_data) load_image(learning_resource, image_data) @@ -544,7 +555,7 @@ def load_program( run.published = False run.save() - load_next_start_date_and_prices(learning_resource) + load_run_dependent_values(learning_resource) for course_data in courses_data: # skip courses that don't define a readable_id diff --git a/learning_resources/etl/loaders_test.py b/learning_resources/etl/loaders_test.py index 905b17b3a1..a33adec777 100644 --- a/learning_resources/etl/loaders_test.py +++ b/learning_resources/etl/loaders_test.py @@ -11,12 +11,13 @@ from django.utils import timezone from learning_resources.constants import ( + Availability, LearningResourceFormat, LearningResourceRelationTypes, LearningResourceType, OfferedBy, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import ( CourseLoaderConfig, @@ -32,7 +33,6 @@ load_courses, load_image, load_instructors, - load_next_start_date_and_prices, load_offered_by, load_playlist, load_playlists, @@ -42,6 +42,7 @@ load_program, load_programs, load_run, + load_run_dependent_values, load_topics, load_video, load_video_channels, @@ -626,7 +627,7 @@ def test_load_course_unique_urls(unique_url): def test_load_course_fetch_only(mocker, course_exists): """When fetch_only is True, course should just be fetched from db""" mock_next_runs_prices = mocker.patch( - "learning_resources.etl.loaders.load_next_start_date_and_prices" + "learning_resources.etl.loaders.load_run_dependent_values" ) mock_warn = mocker.patch("learning_resources.etl.loaders.log.warning") platform = LearningResourcePlatformFactory.create(code=PlatformType.mitpe.name) @@ -653,11 +654,9 @@ def test_load_course_fetch_only(mocker, course_exists): @pytest.mark.parametrize("run_exists", [True, False]) -@pytest.mark.parametrize( - "availability", [RunAvailability.archived.value, RunAvailability.current.value] -) +@pytest.mark.parametrize("status", [RunStatus.archived.value, RunStatus.current.value]) @pytest.mark.parametrize("certification", [True, False]) -def test_load_run(run_exists, availability, certification): +def test_load_run(run_exists, status, certification): """Test that load_run loads the course run""" course = LearningResourceFactory.create( is_course=True, runs=[], certification=certification @@ -670,10 +669,10 @@ def test_load_run(run_exists, availability, certification): props = model_to_dict( LearningResourceRunFactory.build( run_id=learning_resource_run.run_id, - availability=availability, prices=["70.00", "20.00"], ) ) + props["status"] = status del props["id"] del props["learning_resource"] @@ -691,7 +690,7 @@ def test_load_run(run_exists, availability, certification): assert result.prices == ( [] - if (availability == RunAvailability.archived.value or certification is False) + if (status == RunStatus.archived.value or certification is False) else sorted(props["prices"]) ) props.pop("prices") @@ -1461,19 +1460,31 @@ def test_load_course_percolation( @pytest.mark.parametrize("certification", [True, False]) -def test_load_prices_by_certificate(certification): - """Prices should be empty for a course without certificates, else equal to only published run""" +def test_load_run_dependent_values(certification): + """Prices and availability should be correctly assigned based on run data""" course = LearningResourceFactory.create( is_course=True, certification=certification, runs=[] ) + closest_date = now_in_utc() + timedelta(days=1) + furthest_date = now_in_utc() + timedelta(days=2) run = LearningResourceRunFactory.create( learning_resource=course, published=True, - availability=RunAvailability.current.value, + availability=Availability.dated.name, prices=[Decimal("0.00"), Decimal("20.00")], + start_date=closest_date, + ) + LearningResourceRunFactory.create( + learning_resource=course, + published=True, + availability=Availability.dated.name, + prices=[Decimal("0.00"), Decimal("50.00")], + start_date=furthest_date, ) - load_next_start_date_and_prices(course) - assert course.prices == ([] if not certification else run.prices) + result = load_run_dependent_values(course) + assert result.next_start_date == course.next_start_date == closest_date + assert result.prices == course.prices == ([] if not certification else run.prices) + assert result.availability == course.availability == Availability.dated.name @pytest.mark.parametrize( diff --git a/learning_resources/etl/micromasters.py b/learning_resources/etl/micromasters.py index c33477c7e7..b4bec1232e 100644 --- a/learning_resources/etl/micromasters.py +++ b/learning_resources/etl/micromasters.py @@ -90,6 +90,7 @@ def transform(programs_data): or program["enrollment_start"], "end_date": program["end_date"], "enrollment_start": program["enrollment_start"], + "availability": Availability.dated.name, } ], "topics": program["topics"], diff --git a/learning_resources/etl/micromasters_test.py b/learning_resources/etl/micromasters_test.py index 64169557e6..ede2c8df77 100644 --- a/learning_resources/etl/micromasters_test.py +++ b/learning_resources/etl/micromasters_test.py @@ -161,6 +161,7 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "start_date": "2019-10-04T20:13:26.367297Z", "end_date": None, "enrollment_start": "2019-09-29T20:13:26.367297Z", + "availability": Availability.dated.name, } ], "topics": [{"name": "program"}, {"name": "first"}], diff --git a/learning_resources/etl/mitxonline.py b/learning_resources/etl/mitxonline.py index 65ce28c65a..7495bd9f12 100644 --- a/learning_resources/etl/mitxonline.py +++ b/learning_resources/etl/mitxonline.py @@ -15,7 +15,7 @@ LearningResourceType, OfferedBy, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import ETLSource from learning_resources.etl.utils import ( @@ -233,9 +233,10 @@ def _transform_run(course_run: dict, course: dict) -> dict: {"full_name": instructor["name"]} for instructor in parse_page_attribute(course, "instructors", is_list=True) ], - "availability": RunAvailability.current.value + "status": RunStatus.current.value if parse_page_attribute(course, "page_url") - else RunAvailability.archived.value, + else RunStatus.archived.value, + "availability": course.get("availability"), } @@ -366,9 +367,10 @@ def transform_programs(programs): parse_page_attribute(program, "description") ), "prices": parse_program_prices(program), - "availability": RunAvailability.current.value + "status": RunStatus.current.value if parse_page_attribute(program, "page_url") - else RunAvailability.archived.value, + else RunStatus.archived.value, + "availability": program.get("availability"), } ], "courses": transform_courses( diff --git a/learning_resources/etl/mitxonline_test.py b/learning_resources/etl/mitxonline_test.py index ed10ab6824..637cc81209 100644 --- a/learning_resources/etl/mitxonline_test.py +++ b/learning_resources/etl/mitxonline_test.py @@ -13,7 +13,7 @@ CertificationType, LearningResourceType, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import CourseNumberType, ETLSource from learning_resources.etl.mitxonline import ( @@ -167,9 +167,10 @@ def test_mitxonline_transform_programs( program_data.get("page", {}).get("description", None) ), "url": parse_page_attribute(program_data, "page_url", is_url=True), - "availability": RunAvailability.current.value + "status": RunStatus.current.value if parse_page_attribute(program_data, "page_url") - else RunAvailability.archived.value, + else RunStatus.archived.value, + "availability": program_data["availability"], } ], "courses": [ @@ -245,9 +246,10 @@ def test_mitxonline_transform_programs( course_run_data, "instructors", is_list=True ) ], - "availability": RunAvailability.current.value + "status": RunStatus.current.value if parse_page_attribute(course_data, "page_url") - else RunAvailability.archived.value, + else RunStatus.archived.value, + "availability": course_data["availability"], } for course_run_data in course_data["courseruns"] ], @@ -373,9 +375,10 @@ def test_mitxonline_transform_courses(settings, mock_mitxonline_courses_data): course_run_data, "instructors", is_list=True ) ], - "availability": RunAvailability.current.value + "status": RunStatus.current.value if parse_page_attribute(course_data, "page_url") - else RunAvailability.archived.value, + else RunStatus.archived.value, + "availability": course_data["availability"], } for course_run_data in course_data["courseruns"] ], diff --git a/learning_resources/etl/ocw.py b/learning_resources/etl/ocw.py index 12c5a81bbd..59a8b8dd9a 100644 --- a/learning_resources/etl/ocw.py +++ b/learning_resources/etl/ocw.py @@ -24,7 +24,7 @@ LearningResourceType, OfferedBy, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import ETLSource from learning_resources.etl.utils import ( @@ -34,11 +34,7 @@ transform_levels, transform_topics, ) -from learning_resources.models import ( - ContentFile, - LearningResource, - default_learning_format, -) +from learning_resources.models import ContentFile, LearningResource, default_delivery from learning_resources.utils import ( get_s3_object_and_read, parse_instructors, @@ -63,7 +59,7 @@ def parse_delivery(course_data: dict) -> list[str]: Returns: list[str]: The delivery method(s) """ - delivery = default_learning_format() + delivery = default_delivery() if not course_data.get("hide_download"): delivery.append(LearningResourceDelivery.offline.name) return delivery @@ -266,7 +262,7 @@ def transform_run(course_data: dict) -> dict: "description": clean_data(course_data.get("course_description_html")), "year": year, "semester": semester, - "availability": RunAvailability.current.value, + "status": RunStatus.current.value, "image": { "url": urljoin(settings.OCW_BASE_URL, image_src) if image_src else None, "description": course_data.get("course_image_metadata", {}).get( @@ -283,6 +279,8 @@ def transform_run(course_data: dict) -> dict: "title": course_data.get("course_title"), "slug": course_data.get("slug"), "url": course_data["url"], + "availability": Availability.anytime.name, + "delivery": parse_delivery(course_data), } diff --git a/learning_resources/etl/ocw_test.py b/learning_resources/etl/ocw_test.py index b29a7cc00d..39cecd7a4e 100644 --- a/learning_resources/etl/ocw_test.py +++ b/learning_resources/etl/ocw_test.py @@ -9,7 +9,11 @@ from moto import mock_s3 from learning_resources.conftest import OCW_TEST_PREFIX, setup_s3_ocw -from learning_resources.constants import DEPARTMENTS, LearningResourceDelivery +from learning_resources.constants import ( + DEPARTMENTS, + Availability, + LearningResourceDelivery, +) from learning_resources.etl.constants import CourseNumberType, ETLSource from learning_resources.etl.ocw import ( transform_content_files, @@ -240,6 +244,9 @@ def test_transform_course( # noqa: PLR0913 assert transformed_json["runs"][0]["semester"] == (term if term else None) assert transformed_json["runs"][0]["year"] == (year if year else None) assert transformed_json["license_cc"] is True + assert transformed_json["runs"][0]["delivery"] == expected_delivery + assert transformed_json["runs"][0]["availability"] == Availability.anytime.name + assert transformed_json["availability"] == Availability.anytime.name assert transformed_json["description"] == clean_data( course_json["course_description_html"] ) diff --git a/learning_resources/etl/oll.py b/learning_resources/etl/oll.py index 5ac9792ec0..5c720cbafe 100644 --- a/learning_resources/etl/oll.py +++ b/learning_resources/etl/oll.py @@ -15,7 +15,7 @@ Availability, OfferedBy, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import ETLSource from learning_resources.etl.utils import generate_course_numbers_json, transform_levels @@ -44,8 +44,8 @@ def parse_readable_id(course_data: dict, run: dict) -> str: if course_data["Offered by"] == "OCW": semester = run.get("semester") or "" year = run.get("year") or "" - return f"{course_data["OLL Course"]}+{slugify(semester)}_{year}" - return f"MITx+{course_data["OLL Course"]}" + return f"{course_data['OLL Course']}+{slugify(semester)}_{year}" + return f"MITx+{course_data['OLL Course']}" def extract(sheets_id: str or None = None) -> str: @@ -141,7 +141,8 @@ def transform_run(course_data: dict) -> list[dict]: ] if instructor ], - "availability": RunAvailability.archived.value, + "status": RunStatus.archived.value, + "availability": Availability.anytime.name, } ] diff --git a/learning_resources/etl/oll_test.py b/learning_resources/etl/oll_test.py index 44c4458816..f0eebf8ea6 100644 --- a/learning_resources/etl/oll_test.py +++ b/learning_resources/etl/oll_test.py @@ -69,9 +69,10 @@ def test_oll_transform(mocker, oll_course_data): {"full_name": "Jeremy Orloff"}, {"full_name": "Jennifer French Kamrin"}, ], - "availability": "Archived", "semester": "Summer", "year": 2022, + "status": "Archived", + "availability": "anytime", } ], "image": { @@ -120,9 +121,10 @@ def test_oll_transform(mocker, oll_course_data): {"full_name": "Justin Reich"}, {"full_name": "Elizabeth Huttner-Loan"}, ], - "availability": "Archived", "semester": "Spring", "year": 2019, + "status": "Archived", + "availability": "anytime", } ], "image": { diff --git a/learning_resources/etl/openedx.py b/learning_resources/etl/openedx.py index fe01dd9678..c7cc98b532 100644 --- a/learning_resources/etl/openedx.py +++ b/learning_resources/etl/openedx.py @@ -20,7 +20,7 @@ CertificationType, LearningResourceType, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import COMMON_HEADERS from learning_resources.etl.utils import ( @@ -150,7 +150,7 @@ def _get_run_published(course_run): def _get_run_availability(course_run): - if course_run.get("availability") == RunAvailability.archived.value: + if course_run.get("availability") == RunStatus.archived.value: # Enrollable, archived courses can be started anytime return Availability.anytime @@ -165,7 +165,16 @@ def _get_run_availability(course_run): return Availability.dated -def _get_course_availability(course): +def _get_course_availability(course: dict) -> str: + """ + Get the availability of a course based on its runs + + Args: + course (dict): the course data + + Returns: + str: the availability of the course + """ published_runs = [ run for run in course.get("course_runs", []) if _get_run_published(run) ] @@ -178,6 +187,29 @@ def _get_course_availability(course): return None +def _get_program_availability(program: dict) -> str: + """ + Get the availability of a program based on its courses + + Args: + program (dict): the program data + + Returns: + str: the availability of the program + """ + course_availabilities = [ + _get_course_availability(course) for course in program["courses"] + ] + if Availability.dated.name in course_availabilities: + return Availability.dated.name + elif all( + availability == Availability.anytime.name + for availability in course_availabilities + ): + return Availability.anytime.name + return None + + def _is_resource_or_run_deleted(title: str) -> bool: """ Returns True if '[delete]', 'delete ' (note the ending space character) @@ -322,7 +354,8 @@ def _transform_course_run(config, course_run, course_last_modified, marketing_ur "enrollment_start": course_run.get("enrollment_start"), "enrollment_end": course_run.get("enrollment_end"), "image": _transform_course_image(course_run.get("image")), - "availability": course_run.get("availability"), + "status": course_run.get("availability"), + "availability": _get_run_availability(course_run).name, "url": marketing_url or "{}{}/course/".format(config.alt_url, course_run.get("key")), "prices": sorted( @@ -420,10 +453,11 @@ def _transform_program_run( _parse_course_dates(program, "enrollment_end"), default=None ), "image": image, - "availability": RunAvailability.current.value, + "status": RunStatus.current.value, "url": program.get("marketing_url"), "prices": [_sum_course_prices(program)], "instructors": program.pop("instructors", []), + "availability": _get_program_availability(program), } @@ -514,7 +548,7 @@ def _transform_program(config: OpenEdxConfiguration, program: dict) -> dict: "certification_type": CertificationType.completion.name if has_certification else CertificationType.none.name, - "availability": Availability.anytime.name, + "availability": runs[0]["availability"], "courses": [ _transform_program_course(config, course) for course in program.get("courses", []) diff --git a/learning_resources/etl/openedx_test.py b/learning_resources/etl/openedx_test.py index 048e17cf49..3503cc15c4 100644 --- a/learning_resources/etl/openedx_test.py +++ b/learning_resources/etl/openedx_test.py @@ -13,7 +13,7 @@ LearningResourceType, OfferedBy, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import COMMON_HEADERS, CourseNumberType from learning_resources.etl.openedx import ( @@ -223,7 +223,7 @@ def test_transform_course( # noqa: PLR0913 if is_run_deleted or not has_runs else [ { - "availability": "Starting Soon", + "status": "Starting Soon", "run_id": "course-v1:MITx+15.071x+1T2019", "end_date": "2019-05-22T23:30:00Z", "enrollment_end": None, @@ -248,6 +248,7 @@ def test_transform_course( # noqa: PLR0913 "url": "http://localhost/fake-alt-url/this_course", "year": 2019, "published": is_run_enrollable and is_run_published, + "availability": Availability.dated.name, } ] ), @@ -280,7 +281,7 @@ def test_transform_course( # noqa: PLR0913 [ ( { - "availability": RunAvailability.current.value, + "availability": RunStatus.current.value, "pacing_type": "self_paced", "start": "2021-01-01T00:00:00Z", # past }, @@ -288,7 +289,7 @@ def test_transform_course( # noqa: PLR0913 ), ( { - "availability": RunAvailability.current.value, + "availability": RunStatus.current.value, "pacing_type": "self_paced", "start": "2221-01-01T00:00:00Z", # future }, @@ -296,7 +297,7 @@ def test_transform_course( # noqa: PLR0913 ), ( { - "availability": RunAvailability.archived.value, + "availability": RunStatus.archived.value, }, Availability.anytime.name, ), @@ -343,7 +344,7 @@ def test_transform_course_availability_with_multiple_runs( extracted = mitx_course_data["results"] run0 = { # anytime run **extracted[0]["course_runs"][0], - "availability": RunAvailability.current.value, + "availability": RunStatus.current.value, "pacing_type": "self_paced", "start": "2021-01-01T00:00:00Z", # past "is_enrollable": True, @@ -351,13 +352,13 @@ def test_transform_course_availability_with_multiple_runs( } run1 = { # anytime run **extracted[0]["course_runs"][0], - "availability": RunAvailability.archived.value, + "availability": RunStatus.archived.value, "is_enrollable": True, "status": "published", } run2 = { # dated run **extracted[0]["course_runs"][0], - "availability": RunAvailability.current.value, + "availability": RunStatus.current.value, "pacing_type": "instructor_paced", "start": "2221-01-01T00:00:00Z", "is_enrollable": True, @@ -424,7 +425,7 @@ def test_transform_program( "runs": ( [ { - "availability": RunAvailability.current.value, + "status": RunStatus.current.value, "run_id": extracted[0]["uuid"], "start_date": "2019-06-20T15:00:00Z", "end_date": "2025-05-26T15:00:00Z", @@ -448,6 +449,7 @@ def test_transform_program( "title": extracted[0]["title"], "url": extracted[0]["marketing_url"], "published": True, + "availability": Availability.anytime.name, } ] ), diff --git a/learning_resources/etl/prolearn.py b/learning_resources/etl/prolearn.py index 87dd0b404f..063ff6123f 100644 --- a/learning_resources/etl/prolearn.py +++ b/learning_resources/etl/prolearn.py @@ -322,6 +322,8 @@ def _transform_runs(resource: dict) -> list[dict]: "published": True, "prices": parse_price(resource), "url": parse_url(resource), + "delivery": transform_delivery(resource["format_name"]), + "availability": Availability.dated.name, } ) return runs @@ -357,6 +359,7 @@ def _transform_course( "course_numbers": [], }, "learning_format": transform_delivery(course["format_name"]), + "delivery": transform_delivery(course["format_name"]), "published": True, "topics": parse_topic(course, offered_by.code) if offered_by else None, "runs": runs, diff --git a/learning_resources/etl/prolearn_test.py b/learning_resources/etl/prolearn_test.py index b050f17949..a2f9619ac3 100644 --- a/learning_resources/etl/prolearn_test.py +++ b/learning_resources/etl/prolearn_test.py @@ -176,6 +176,8 @@ def test_prolearn_transform_programs(mock_csail_programs_data): or program["course_application_url"] or urljoin(PROLEARN_BASE_URL, program["url"]) ), + "availability": Availability.dated.name, + "delivery": transform_delivery(program["format_name"]), } for (start_val, end_val) in zip( program["start_value"], program["end_value"] @@ -252,6 +254,8 @@ def test_prolearn_transform_courses(mock_mitpe_courses_data): or course["course_application_url"] or urljoin(PROLEARN_BASE_URL, course["url"]) ), + "availability": Availability.dated.name, + "delivery": transform_delivery(course["format_name"]), } for (start_val, end_val) in zip( course["start_value"], course["end_value"] diff --git a/learning_resources/etl/sloan.py b/learning_resources/etl/sloan.py index 1446d9a479..496a155a75 100644 --- a/learning_resources/etl/sloan.py +++ b/learning_resources/etl/sloan.py @@ -14,7 +14,7 @@ CertificationType, OfferedBy, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import ETLSource from learning_resources.etl.utils import ( @@ -106,23 +106,21 @@ def parse_datetime(value): ) -def parse_availability(runs_data): +def parse_availability(run_data: dict) -> str: """ - Parse availability from runs data + Parse availability from run data Args: - runs_data (list): the runs data + run_data (list): the run data Returns: str: the availability """ - if runs_data: - run = runs_data[0] - if ( - run.get("Delivery", "") == "Online" - and run.get("Format", "") == "Asynchronous (On-Demand)" - ): - return Availability.anytime.name + if run_data and ( + run_data.get("Delivery", "") == "Online" + and run_data.get("Format", "") == "Asynchronous (On-Demand)" + ): + return Availability.anytime.name return Availability.dated.name @@ -180,7 +178,9 @@ def transform_run(run_data, course_data): "end_date": parse_datetime(run_data["End_Date"]), "title": course_data["Title"], "url": course_data["URL"], - "availability": RunAvailability.current.value, + "status": RunStatus.current.value, + "delivery": transform_delivery(run_data["Delivery"]), + "availability": parse_availability(run_data), "published": True, "prices": [run_data["Price"]], "instructors": [{"full_name": name.strip()} for name in faculty_names], @@ -225,7 +225,6 @@ def transform_course(course_data: dict, runs_data: dict) -> dict: "course_numbers": [], }, "runs": [transform_run(run, course_data) for run in course_runs_data], - "availability": parse_availability(course_runs_data), "continuing_ed_credits": course_runs_data[0]["Continuing_Ed_Credits"], } diff --git a/learning_resources/etl/sloan_test.py b/learning_resources/etl/sloan_test.py index d7f4ff408a..72a65bf91e 100644 --- a/learning_resources/etl/sloan_test.py +++ b/learning_resources/etl/sloan_test.py @@ -7,7 +7,7 @@ from learning_resources.constants import ( Availability, - RunAvailability, + RunStatus, ) from learning_resources.etl.sloan import ( extract, @@ -118,7 +118,9 @@ def test_transform_run( "end_date": parse_datetime(run_data["End_Date"]), "title": course_data["Title"], "url": course_data["URL"], - "availability": RunAvailability.current.value, + "status": RunStatus.current.value, + "delivery": transform_delivery(run_data["Delivery"]), + "availability": parse_availability(run_data), "published": True, "prices": [run_data["Price"]], "instructors": [{"full_name": name.strip()} for name in faculty_names], @@ -127,12 +129,13 @@ def test_transform_run( def test_transform_course(mock_sloan_courses_data, mock_sloan_runs_data): course_data = mock_sloan_courses_data[0] - runs_data = mock_sloan_runs_data[0:10] course_runs_data = [ - run for run in runs_data if run["Course_Id"] == course_data["Course_Id"] + run + for run in mock_sloan_runs_data + if run["Course_Id"] == course_data["Course_Id"] ] - transformed = transform_course(course_data, runs_data) + transformed = transform_course(course_data, mock_sloan_runs_data) assert transformed["readable_id"] == course_data["Course_Id"] assert transformed["runs"] == [ @@ -142,8 +145,10 @@ def test_transform_course(mock_sloan_courses_data, mock_sloan_runs_data): {transform_delivery(run["Delivery"])[0] for run in course_runs_data} ) assert transformed["delivery"] == transformed["learning_format"] + assert transformed["runs"][0]["availability"] == parse_availability( + course_runs_data[0] + ) assert transformed["image"] == parse_image(course_data) - assert transformed["availability"] == parse_availability(course_runs_data) assert ( transformed["continuing_ed_credits"] == course_runs_data[0]["Continuing_Ed_Credits"] @@ -167,7 +172,6 @@ def test_transform_course(mock_sloan_courses_data, mock_sloan_runs_data): "topics", "course", "runs", - "availability", "continuing_ed_credits", ] ) @@ -209,14 +213,12 @@ def test_parse_topics(mock_sloan_courses_data, mock_sloan_runs_data): ], ) def test_parse_availability(delivery, run_format, availability): - runs_data = [ - { - "Format": run_format, - "Delivery": delivery, - } - ] - assert parse_availability(runs_data) == availability - assert parse_availability([]) == Availability.dated.name + run_data = { + "Format": run_format, + "Delivery": delivery, + } + assert parse_availability(run_data) == availability + assert parse_availability(None) == Availability.dated.name def test_enabled_flag(mock_sloan_api_setting, settings): diff --git a/learning_resources/etl/utils.py b/learning_resources/etl/utils.py index f0871b243d..e8e59a37bf 100644 --- a/learning_resources/etl/utils.py +++ b/learning_resources/etl/utils.py @@ -34,7 +34,7 @@ LearningResourceFormat, LevelType, OfferedBy, - RunAvailability, + RunStatus, ) from learning_resources.etl.constants import ( RESOURCE_FORMAT_MAPPING, @@ -706,18 +706,16 @@ def transform_delivery(resource_delivery: str) -> list[str]: def parse_certification(offeror, runs_data): - """Return true/false depending on offeror and run availability""" + """Return true/false depending on offeror and run status""" if offeror != OfferedBy.mitx.name: return False return bool( [ - availability - for availability in [ - run.get("availability") - for run in runs_data - if run.get("published", True) + status + for status in [ + run.get("status") for run in runs_data if run.get("published", True) ] - if (availability and availability != RunAvailability.archived.value) + if (status and status != RunStatus.archived.value) ] ) diff --git a/learning_resources/etl/utils_test.py b/learning_resources/etl/utils_test.py index 6a3f37a694..ab4bd3fa43 100644 --- a/learning_resources/etl/utils_test.py +++ b/learning_resources/etl/utils_test.py @@ -17,7 +17,7 @@ LearningResourceType, OfferedBy, PlatformType, - RunAvailability, + RunStatus, ) from learning_resources.etl import utils from learning_resources.etl.utils import parse_certification @@ -384,50 +384,48 @@ def test_parse_bad_format(mocker): @pytest.mark.parametrize( - ("offered_by", "availability", "has_cert"), + ("offered_by", "status", "has_cert"), [ - [ # noqa: PT007 + ( OfferedBy.ocw.name, - RunAvailability.archived.value, + RunStatus.archived.value, False, - ], - [ # noqa: PT007 + ), + ( OfferedBy.ocw.name, - RunAvailability.current.value, + RunStatus.current.value, False, - ], - [ # noqa: PT007 + ), + ( OfferedBy.mitx.name, - RunAvailability.archived.value, + RunStatus.archived.value, False, - ], - [ # noqa: PT007 + ), + ( OfferedBy.mitx.name, - RunAvailability.current.value, + RunStatus.current.value, True, - ], - [ # noqa: PT007 + ), + ( OfferedBy.mitx.name, - RunAvailability.upcoming.value, + RunStatus.upcoming.value, True, - ], + ), ], ) -def test_parse_certification(offered_by, availability, has_cert): +def test_parse_certification(offered_by, status, has_cert): """The parse_certification function should return the expected bool value""" offered_by_obj = LearningResourceOfferorFactory.create(code=offered_by) resource = LearningResourceRunFactory.create( - availability=availability, learning_resource=LearningResourceFactory.create( published=True, resource_type=LearningResourceType.podcast.name, offered_by=offered_by_obj, ), ).learning_resource - assert resource.runs.first().availability == availability assert resource.runs.count() == 1 - runs = resource.runs.all().values() + runs = [{"status": status, **run} for run in resource.runs.all().values()] assert parse_certification(offered_by_obj.code, runs) == has_cert diff --git a/learning_resources/etl/xpro.py b/learning_resources/etl/xpro.py index b620473dea..2393d2e703 100644 --- a/learning_resources/etl/xpro.py +++ b/learning_resources/etl/xpro.py @@ -87,16 +87,17 @@ def extract_courses(): return [] -def _transform_run(course_run): +def _transform_run(course_run: dict, course: dict) -> dict: """ - Transforms a course run into our normalized data structure + Transform a course run into our normalized data structure Args: course_run (dict): course run data + course (dict): course data Returns: dict: normalized course run data - """ # noqa: D401 + """ return { "run_id": course_run["courseware_id"], "title": course_run["title"], @@ -108,14 +109,14 @@ def _transform_run(course_run): "enrollment_end": _parse_datetime(course_run["enrollment_end"]), "published": bool(course_run["current_price"]), "prices": ( - [course_run["current_price"]] - if course_run.get("current_price", None) - else [] + [course_run["current_price"]] if course_run.get("current_price") else [] ), "instructors": [ {"full_name": instructor["name"]} for instructor in course_run["instructors"] ], + "availability": course["availability"], + "delivery": transform_delivery(course.get("format")), } @@ -143,7 +144,9 @@ def _transform_learning_resource_course(course): course_run.get("current_price", None) for course_run in course["courseruns"] ), "topics": parse_topics(course), - "runs": [_transform_run(course_run) for course_run in course["courseruns"]], + "runs": [ + _transform_run(course_run, course) for course_run in course["courseruns"] + ], "resource_type": LearningResourceType.course.name, "learning_format": transform_delivery(course.get("format")), "delivery": transform_delivery(course.get("format")), @@ -213,6 +216,8 @@ def transform_programs(programs): {"full_name": instructor["name"]} for instructor in program.get("instructors", []) ], + "delivery": transform_delivery(program.get("format")), + "availability": program["availability"], } ], "courses": transform_courses(program["courses"]), diff --git a/learning_resources/etl/xpro_test.py b/learning_resources/etl/xpro_test.py index 25ffdbbe9d..2013deeb51 100644 --- a/learning_resources/etl/xpro_test.py +++ b/learning_resources/etl/xpro_test.py @@ -130,6 +130,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): for instructor in program_data.get("instructors", []) ], "description": program_data["description"], + "delivery": transform_delivery(program_data.get("format")), + "availability": Availability.dated.name, } ], "courses": [ @@ -171,6 +173,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): {"full_name": instructor["name"]} for instructor in course_run_data["instructors"] ], + "delivery": transform_delivery(course_data.get("format")), + "availability": Availability.dated.name, } for course_run_data in course_data["courseruns"] ], @@ -242,6 +246,8 @@ def test_xpro_transform_courses(mock_xpro_courses_data): {"full_name": instructor["name"]} for instructor in course_run_data["instructors"] ], + "delivery": transform_delivery(course_data.get("format")), + "availability": Availability.dated.name, } for course_run_data in course_data["courseruns"] ], diff --git a/learning_resources/factories.py b/learning_resources/factories.py index 415c853690..33008e8f14 100644 --- a/learning_resources/factories.py +++ b/learning_resources/factories.py @@ -492,14 +492,7 @@ class LearningResourceRunFactory(DjangoModelFactory): languages = factory.List(random.choices(["en", "es"])) # noqa: S311 year = factory.Faker("year") image = factory.SubFactory(LearningResourceImageFactory) - availability = FuzzyChoice( - ( - constants.RunAvailability.current.value, - constants.RunAvailability.upcoming.value, - constants.RunAvailability.starting_soon.value, - constants.RunAvailability.archived.value, - ) - ) + availability = FuzzyChoice(Availability.names()) enrollment_start = factory.Faker("future_datetime", tzinfo=UTC) enrollment_end = factory.LazyAttribute( lambda obj: ( diff --git a/learning_resources/migrations/0046_learningresource_certification.py b/learning_resources/migrations/0046_learningresource_certification.py index 64de2d1076..be555dfb08 100644 --- a/learning_resources/migrations/0046_learningresource_certification.py +++ b/learning_resources/migrations/0046_learningresource_certification.py @@ -5,7 +5,7 @@ from learning_resources.constants import ( LearningResourceType, OfferedBy, - RunAvailability, + RunStatus, ) @@ -27,7 +27,7 @@ def populate_certification(apps, schema_editor): and lr.offered_by.name == OfferedBy.mitx.value and ( any( - availability != RunAvailability.archived.value + availability != RunStatus.archived.value for availability in lr.runs.values_list( "availability", flat=True ) diff --git a/learning_resources/migrations/0062_learningresource_delivery_format_pace.py b/learning_resources/migrations/0062_learningresource_delivery_format_pace.py index 5318553c48..4120595f03 100644 --- a/learning_resources/migrations/0062_learningresource_delivery_format_pace.py +++ b/learning_resources/migrations/0062_learningresource_delivery_format_pace.py @@ -137,7 +137,7 @@ class Migration(migrations.Migration): db_index=True, max_length=24, ), - default=learning_resources.models.default_learning_format, + default=learning_resources.models.default_delivery, size=None, ), ), diff --git a/learning_resources/migrations/0064_learningresourcerun_delivery.py b/learning_resources/migrations/0064_learningresourcerun_delivery.py new file mode 100644 index 0000000000..d60f143dcf --- /dev/null +++ b/learning_resources/migrations/0064_learningresourcerun_delivery.py @@ -0,0 +1,55 @@ +# Generated by Django 4.2.15 on 2024-08-26 21:37 + +import django.contrib.postgres.fields +from django.db import migrations, models + +import learning_resources.models +from learning_resources.constants import LearningResourceType + + +def assign_run_delivery(apps, schema_editor): + """Assign initial delivery method to all runs based on the parent resource""" + LearningResource = apps.get_model("learning_resources", "LearningResource") + for resource in LearningResource.objects.filter( + resource_type__in=[ + LearningResourceType.course.name, + LearningResourceType.program.name, + ], + published=True, + ): + resource.runs.filter(published=True).update(delivery=resource.delivery) + + +class Migration(migrations.Migration): + dependencies = [ + ("learning_resources", "0063_continuing_ed_credits_cc_license"), + ] + + operations = [ + migrations.RemoveField( + model_name="learningresourcerun", + name="availability", + ), + migrations.AddField( + model_name="learningresourcerun", + name="delivery", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("online", "Online"), + ("hybrid", "Hybrid"), + ("in_person", "In person"), + ("offline", "Offline"), + ], + db_index=True, + max_length=24, + ), + default=learning_resources.models.default_delivery, + size=None, + ), + ), + migrations.RunPython( + assign_run_delivery, + reverse_code=migrations.RunPython.noop, + ), + ] diff --git a/learning_resources/migrations/0065_learningresourcerun_availability.py b/learning_resources/migrations/0065_learningresourcerun_availability.py new file mode 100644 index 0000000000..fcd50444a6 --- /dev/null +++ b/learning_resources/migrations/0065_learningresourcerun_availability.py @@ -0,0 +1,36 @@ +# Generated by Django 4.2.15 on 2024-08-26 21:39 + +from django.db import migrations, models + +from learning_resources.constants import LearningResourceType + + +def assign_run_availability(apps, schema_editor): + """Assign initial availability to all runs based on the parent resource""" + LearningResource = apps.get_model("learning_resources", "LearningResource") + for resource in LearningResource.objects.filter( + resource_type__in=[ + LearningResourceType.course.name, + LearningResourceType.program.name, + ], + published=True, + ): + resource.runs.filter(published=True).update(availability=resource.availability) + + +class Migration(migrations.Migration): + dependencies = [ + ("learning_resources", "0064_learningresourcerun_delivery"), + ] + + operations = [ + migrations.AddField( + model_name="learningresourcerun", + name="availability", + field=models.CharField( + choices=[("dated", "Dated"), ("anytime", "Anytime")], + max_length=24, + null=True, + ), + ) + ] diff --git a/learning_resources/models.py b/learning_resources/models.py index 01446e8dd7..5ddc558948 100644 --- a/learning_resources/models.py +++ b/learning_resources/models.py @@ -26,10 +26,15 @@ def default_learning_format(): - """Return the default learning format list""" + """Return the default learning_format as a list""" return [LearningResourceFormat.online.name] +def default_delivery(): + """Return the default delivery as a list""" + return [LearningResourceDelivery.online.name] + + class LearningResourcePlatform(TimestampedModel): """Platforms for all learning resources""" @@ -427,7 +432,7 @@ class LearningResource(TimestampedModel): models.CharField( max_length=24, db_index=True, choices=LearningResourceDelivery.as_tuple() ), - default=default_learning_format, + default=default_delivery, ) license_cc = models.BooleanField(default=False) continuing_ed_credits = models.DecimalField( @@ -546,9 +551,6 @@ class LearningResourceRun(TimestampedModel): models.CharField(max_length=128), null=False, blank=False, default=list ) slug = models.CharField(max_length=1024, null=True, blank=True) # noqa: DJ001 - availability = models.CharField( # noqa: DJ001 - max_length=128, null=True, blank=True - ) semester = models.CharField(max_length=20, null=True, blank=True) # noqa: DJ001 year = models.IntegerField(null=True, blank=True) start_date = models.DateTimeField(null=True, blank=True) @@ -562,6 +564,17 @@ class LearningResourceRun(TimestampedModel): models.DecimalField(decimal_places=2, max_digits=12), null=True, blank=True ) checksum = models.CharField(max_length=32, null=True, blank=True) # noqa: DJ001 + delivery = ArrayField( + models.CharField( + max_length=24, db_index=True, choices=LearningResourceDelivery.as_tuple() + ), + default=default_delivery, + ) + availability = models.CharField( # noqa: DJ001 + max_length=24, + null=True, + choices=Availability.as_tuple(), + ) def __str__(self): return f"LearningResourceRun platform={self.learning_resource.platform} run_id={self.run_id}" # noqa: E501 diff --git a/learning_resources/serializers.py b/learning_resources/serializers.py index b2be2cc74b..667e639d56 100644 --- a/learning_resources/serializers.py +++ b/learning_resources/serializers.py @@ -261,10 +261,13 @@ class LearningResourceRunSerializer(serializers.ModelSerializer): image = LearningResourceImageSerializer(read_only=True, allow_null=True) level = serializers.ListField(child=LearningResourceLevelSerializer()) + delivery = serializers.ListField( + child=LearningResourceDeliverySerializer(), read_only=True + ) class Meta: model = models.LearningResourceRun - exclude = ["learning_resource", "availability", *COMMON_IGNORED_FIELDS] + exclude = ["learning_resource", *COMMON_IGNORED_FIELDS] class ResourceListMixin(serializers.Serializer): diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py index b36f70487c..3657875b89 100644 --- a/learning_resources/tasks.py +++ b/learning_resources/tasks.py @@ -17,7 +17,7 @@ get_most_recent_course_archives, sync_edx_course_files, ) -from learning_resources.etl.loaders import load_next_start_date_and_prices +from learning_resources.etl.loaders import load_run_dependent_values from learning_resources.etl.pipelines import ocw_courses_etl from learning_resources.etl.utils import get_learning_course_bucket_name from learning_resources.models import LearningResource @@ -34,7 +34,7 @@ def update_next_start_date_and_prices(): """Update expired next start dates and prices""" resources = LearningResource.objects.filter(next_start_date__lt=timezone.now()) for resource in resources: - load_next_start_date_and_prices(resource) + load_run_dependent_values(resource) clear_search_cache() return len(resources) diff --git a/learning_resources/tasks_test.py b/learning_resources/tasks_test.py index 3e15814c58..7739374ae4 100644 --- a/learning_resources/tasks_test.py +++ b/learning_resources/tasks_test.py @@ -377,7 +377,7 @@ def test_update_next_start_date(mocker): LearningResourceFactory.create(next_start_date=(timezone.now() + timedelta(1))) mock_load_next_start_date = mocker.patch( - "learning_resources.tasks.load_next_start_date_and_prices" + "learning_resources.tasks.load_run_dependent_values" ) update_next_start_date_and_prices() mock_load_next_start_date.assert_called_once_with(learning_resource) diff --git a/learning_resources_search/constants.py b/learning_resources_search/constants.py index 1347806875..4b2220d150 100644 --- a/learning_resources_search/constants.py +++ b/learning_resources_search/constants.py @@ -232,6 +232,13 @@ class FilterConfig: "languages": {"type": "keyword"}, "slug": {"type": "keyword"}, "availability": {"type": "keyword"}, + "delivery": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, "semester": {"type": "keyword"}, "year": {"type": "keyword"}, "start_date": {"type": "date"}, diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index 4d4a278a5e..96eb2e9b0c 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -9276,6 +9276,23 @@ components: required: - code - name + delivery: + type: array + items: + type: object + properties: + code: + enum: + - online + - hybrid + - in_person + - offline + name: + type: string + required: + - code + - name + readOnly: true run_id: type: string maxLength: 128 @@ -9345,7 +9362,13 @@ components: type: string nullable: true maxLength: 32 + availability: + nullable: true + oneOf: + - $ref: '#/components/schemas/AvailabilityEnum' + - $ref: '#/components/schemas/NullEnum' required: + - delivery - id - image - instructors @@ -9448,6 +9471,11 @@ components: type: string nullable: true maxLength: 32 + availability: + nullable: true + oneOf: + - $ref: '#/components/schemas/AvailabilityEnum' + - $ref: '#/components/schemas/NullEnum' required: - level - run_id diff --git a/test_json/test_sloan_courses.json b/test_json/test_sloan_courses.json index 9004fcea30..27b0672367 100644 --- a/test_json/test_sloan_courses.json +++ b/test_json/test_sloan_courses.json @@ -767,18 +767,6 @@ "URL": "https://executive.mit.edu/course/blockchain-technologies/a056g00000URaa7AAD.html", "Certification_Type": "Digital Business" }, - { - "Title": "Future Family Enterprise: Sustaining Multigenerational Success", - "Course_Id": "a056g00000URaa8AAD", - "SourceCreateDate": "2018-04-25T09:25:10.000Z", - "SourceLastModifiedDate": "2024-08-09T13:38:31.000Z", - "Description": "Recent research indicates that most families that achieve financial success—typically through a family company—lose their success within three generations. Why do some family enterprises derail while others prosper? This program for multigenerational families will help you understand and implement the important driving factors of long-term, enduring, family enterprise success. The course also examines where family enterprises are going and what will drive their success in the future economy.", - "Topics": "Business: Other Business", - "Image_Src": "https://executive.mit.edu/on/demandware.static/-/Sites-master-catalog-msee/default/vbca8e724fc8fc64d818a33a83f0f6609ebb7dee3/images/FAM.jpg", - "BrochureLink": "https://executive.mit.edu/on/demandware.static/-/Sites-master-catalog-msee/default/dwd70b8935/brochures/Future%20Family%20Enterprise.pdf", - "URL": "https://executive.mit.edu/course/Future-Family-Enterprise/a056g00000URaa8AAD.html", - "Certification_Type": "Management and Leadership" - }, { "Title": "Cybersecurity Leadership for Non-Technical Executives", "Course_Id": "a056g00000URaaBAAT",