From d7209f6d30bc3e56061f64218b7256a88eeaf58d Mon Sep 17 00:00:00 2001 From: Shankar Ambady Date: Thu, 19 Sep 2024 14:31:11 -0400 Subject: [PATCH 01/13] Codespace opensearch service fix (#1582) * testing codespace fix * testing fix --- docker-compose.codespaces.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.codespaces.yml b/docker-compose.codespaces.yml index 965f705a7e..8423ccce38 100644 --- a/docker-compose.codespaces.yml +++ b/docker-compose.codespaces.yml @@ -7,6 +7,8 @@ services: file: docker-compose.apps.yml service: web env_file: env/codespaces.env + depends_on: + - opensearch-node-mitopen-1 watch: extends: From 590bddd616071b81f43e573cdd6741e795e2bb2d Mon Sep 17 00:00:00 2001 From: Doof Date: Thu, 19 Sep 2024 19:25:01 +0000 Subject: [PATCH 02/13] Release 0.19.2 --- RELEASE.rst | 9 +++++++++ main/settings.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index 8c6a4bda8a..e8739530e8 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,15 @@ Release Notes ============= +Version 0.19.2 +-------------- + +- Codespace opensearch service fix (#1582) +- Remove learning_format, use delivery for search filter/facet (#1567) +- Signup popover owns url (#1576) +- [pre-commit.ci] pre-commit autoupdate (#1541) +- Remove an unused dependency (#1571) + Version 0.19.1 (Released September 18, 2024) -------------- diff --git a/main/settings.py b/main/settings.py index 3b3adc4d56..e3038ed7a8 100644 --- a/main/settings.py +++ b/main/settings.py @@ -33,7 +33,7 @@ from main.settings_pluggy import * # noqa: F403 from openapi.settings_spectacular import open_spectacular_settings -VERSION = "0.19.1" +VERSION = "0.19.2" log = logging.getLogger() From 1a1f5a8d4174e8da62cb372744108cc2a0f31478 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Fri, 20 Sep 2024 11:29:17 -0400 Subject: [PATCH 03/13] Fix extract_openedx_data and backpopulate_mit_edx_data commands to work with course/program datafiles (#1587) --- .../commands/backpopulate_mit_edx_data.py | 16 ++++++++++++---- .../management/commands/extract_openedx_data.py | 3 ++- learning_resources/tasks.py | 15 +++++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/learning_resources/management/commands/backpopulate_mit_edx_data.py b/learning_resources/management/commands/backpopulate_mit_edx_data.py index 3886cfe64b..77e183e185 100644 --- a/learning_resources/management/commands/backpopulate_mit_edx_data.py +++ b/learning_resources/management/commands/backpopulate_mit_edx_data.py @@ -22,9 +22,15 @@ def add_arguments(self, parser): help="Delete all existing records first", ) parser.add_argument( - "--api_datafile", - dest="api_datafile", - help="If provided, use this file as the source of API data", + "--api_course_datafile", + dest="api_course_datafile", + help="If provided, use this file as the source of course API data", + default=None, + ) + parser.add_argument( + "--api_program_datafile", + dest="api_program_datafile", + help="If provided, use this file as the source of program API data", default=None, ) super().add_arguments(parser) @@ -40,7 +46,9 @@ def handle(self, *args, **options): # noqa: ARG002 ): resource_delete_actions(learning_resource) else: - task = get_mit_edx_data.delay(options["api_datafile"]) + task = get_mit_edx_data.delay( + options["api_course_datafile"], options["api_program_datafile"] + ) self.stdout.write(f"Started task {task} to get MIT edX course data") self.stdout.write("Waiting on task...") start = now_in_utc() diff --git a/learning_resources/management/commands/extract_openedx_data.py b/learning_resources/management/commands/extract_openedx_data.py index b7f14d3fe1..84cde80f33 100644 --- a/learning_resources/management/commands/extract_openedx_data.py +++ b/learning_resources/management/commands/extract_openedx_data.py @@ -5,13 +5,14 @@ from django.core.management import BaseCommand -from learning_resources.etl import mit_edx, oll +from learning_resources.etl import mit_edx, mit_edx_programs, oll from learning_resources.etl.constants import ETLSource from main.utils import now_in_utc EXTRACTORS = { ETLSource.oll.name: oll.extract, ETLSource.mit_edx.name: mit_edx.extract, + f"{ETLSource.mit_edx.name}_programs": mit_edx_programs.extract, } diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py index 3657875b89..0fd1527a3e 100644 --- a/learning_resources/tasks.py +++ b/learning_resources/tasks.py @@ -48,15 +48,22 @@ def get_micromasters_data(): @app.task -def get_mit_edx_data(api_datafile=None) -> int: +def get_mit_edx_data( + api_course_datafile: str | None = None, api_program_datafile: str | None = None +) -> int: """Task to sync MIT edX data with the database Args: - api_datafile (str): If provided, use this file as the source of API data + api_course_datafile (str): If provided, use file as source of course API data Otherwise, the API is queried directly. + api_program_datafile (str): If provided, use file as source of program API data. + Otherwise, the API is queried directly. + + Returns: + int: The number of results that were fetched """ - courses = pipelines.mit_edx_courses_etl(api_datafile) - programs = pipelines.mit_edx_programs_etl(api_datafile) + courses = pipelines.mit_edx_courses_etl(api_course_datafile) + programs = pipelines.mit_edx_programs_etl(api_program_datafile) clear_search_cache() return len(courses) + len(programs) From c849ff9bb73e31283bc32681fa113f9ec1e04b7f Mon Sep 17 00:00:00 2001 From: Shankar Ambady Date: Mon, 23 Sep 2024 11:46:36 -0400 Subject: [PATCH 04/13] follow/unfollow popover (#1589) * adding success variant * adding working version * adding working version * fixing text and adding margin to buttons * lint fixes * lint fix * fixing test cases * closing popup before posting data * removing redundancy * removing redundancy * removing empty test * updating styles * updating styles * changes to match design * changes to match design --- .../FollowPopover/FollowPopover.tsx | 144 ++++++++++++++++++ .../SearchSubscriptionToggle.test.tsx | 7 +- .../SearchSubscriptionToggle.tsx | 64 ++++---- .../SignupPopover/SignupPopover.tsx | 2 + .../src/components/Button/Button.tsx | 22 +++ .../src/components/Popover/Popover.tsx | 1 + 6 files changed, 202 insertions(+), 38 deletions(-) create mode 100644 frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx diff --git a/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx b/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx new file mode 100644 index 0000000000..3d21398f11 --- /dev/null +++ b/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx @@ -0,0 +1,144 @@ +import React, { useMemo } from "react" +import { Popover, Typography, styled, Button } from "ol-components" +import type { PopoverProps } from "ol-components" +import { getSearchParamMap } from "@/common/utils" + +import { SignupPopover } from "../SignupPopover/SignupPopover" + +import { useUserMe } from "api/hooks/user" +import { SourceTypeEnum } from "api" +import { + useSearchSubscriptionCreate, + useSearchSubscriptionDelete, + useSearchSubscriptionList, +} from "api/hooks/searchSubscription" + +const StyledPopover = styled(Popover)({ + width: "300px", + maxWidth: "100vw", +}) +const HeaderText = styled(Typography)(({ theme }) => ({ + color: theme.custom.colors.darkGray2, + marginBottom: "8px", + ...theme.typography.subtitle2, +})) +const BodyText = styled(Typography)(({ theme }) => ({ + ...theme.typography.body2, + color: theme.custom.colors.silverGrayDark, + marginBottom: "16px", +})) + +const Footer = styled.div({ + display: "flex", + justifyContent: "end", + gap: "16px", +}) + +interface FollowPopoverProps + extends Pick { + itemName?: string + searchParams: URLSearchParams + sourceType: SourceTypeEnum +} + +const FollowPopover: React.FC = ({ + itemName, + searchParams, + sourceType, + ...props +}) => { + const { data: user } = useUserMe() + const subscribeParams: Record = useMemo(() => { + return { source_type: sourceType, ...getSearchParamMap(searchParams) } + }, [searchParams, sourceType]) + + const subscriptionDelete = useSearchSubscriptionDelete() + const subscriptionCreate = useSearchSubscriptionCreate() + const subscriptionList = useSearchSubscriptionList(subscribeParams, { + enabled: !!user?.is_authenticated, + }) + const unsubscribe = subscriptionDelete.mutate + const subscriptionId = subscriptionList.data?.[0]?.id + + const isSubscribed = !!subscriptionId + const handleFollowAction = async (): Promise => { + props.onClose() + if (!isSubscribed) { + await subscriptionCreate.mutateAsync({ + PercolateQuerySubscriptionRequestRequest: subscribeParams, + }) + } else { + unsubscribe(subscriptionId) + } + } + + if (user?.is_authenticated && subscriptionList.isLoading) return null + if (!user) return null + if (!user?.is_authenticated) { + return + } + + if (isSubscribed) { + return ( + + + You are following {itemName} + + + Unfollow to stop getting emails for new {itemName} courses. + +
+ + +
+ + ) + } + return ( + + Follow {itemName}? + + You will get an email when new courses are available. + +
+ + +
+ + ) +} + +export { FollowPopover } +export type { FollowPopoverProps } diff --git a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx index c3116e8a82..844e36e2eb 100644 --- a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx +++ b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx @@ -79,6 +79,8 @@ test.each(Object.values(SourceTypeEnum))( }) await user.click(subscribeButton) + const followButton = screen.getByTestId("action-follow") + await user.click(followButton) expect(makeRequest).toHaveBeenCalledWith("post", subscribeUrl, { source_type: sourceType, offered_by: ["ocw"], @@ -111,11 +113,8 @@ test.each(Object.values(SourceTypeEnum))( }) await user.click(subscribedButton) + const unsubscribeButton = screen.getByTestId("action-unfollow") - const menu = screen.getByRole("menu") - const unsubscribeButton = within(menu).getByRole("menuitem", { - name: "Unfollow", - }) await user.click(unsubscribeButton) expect(makeRequest).toHaveBeenCalledWith( diff --git a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx index b80f87e884..503b1f4cc4 100644 --- a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx +++ b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx @@ -2,15 +2,14 @@ import React, { useState, useMemo } from "react" import { getSearchParamMap } from "@/common/utils" import { useSearchSubscriptionCreate, - useSearchSubscriptionDelete, useSearchSubscriptionList, } from "api/hooks/searchSubscription" -import { Button, SimpleMenu, styled } from "ol-components" -import type { SimpleMenuItem } from "ol-components" -import { RiArrowDownSLine, RiMailLine } from "@remixicon/react" +import { Button, styled } from "ol-components" + +import { RiMailLine } from "@remixicon/react" import { useUserMe } from "api/hooks/user" import { SourceTypeEnum } from "api" -import { SignupPopover } from "../SignupPopover/SignupPopover" +import { FollowPopover } from "../FollowPopover/FollowPopover" const StyledButton = styled(Button)({ minWidth: "130px", @@ -23,6 +22,7 @@ type SearchSubscriptionToggleProps = { } const SearchSubscriptionToggle: React.FC = ({ + itemName, searchParams, sourceType, }) => { @@ -33,35 +33,17 @@ const SearchSubscriptionToggle: React.FC = ({ }, [searchParams, sourceType]) const { data: user } = useUserMe() - const subscriptionDelete = useSearchSubscriptionDelete() + const subscriptionCreate = useSearchSubscriptionCreate() const subscriptionList = useSearchSubscriptionList(subscribeParams, { enabled: !!user?.is_authenticated, }) - const unsubscribe = subscriptionDelete.mutate const subscriptionId = subscriptionList.data?.[0]?.id const isSubscribed = !!subscriptionId - const unsubscribeItems: SimpleMenuItem[] = useMemo(() => { - if (!subscriptionId) return [] - return [ - { - key: "unsubscribe", - label: "Unfollow", - onClick: () => unsubscribe(subscriptionId), - }, - ] - }, [unsubscribe, subscriptionId]) - const onFollowClick = async (event: React.MouseEvent) => { - if (user?.is_authenticated) { - await subscriptionCreate.mutateAsync({ - PercolateQuerySubscriptionRequestRequest: subscribeParams, - }) - } else { - setButtonEl(event.currentTarget) - } + setButtonEl(event.currentTarget) } if (user?.is_authenticated && subscriptionList.isLoading) return null @@ -69,14 +51,22 @@ const SearchSubscriptionToggle: React.FC = ({ if (isSubscribed) { return ( - }> - Following - - } - items={unsubscribeItems} - /> + <> + } + > + Following + + setButtonEl(null)} + /> + ) } @@ -90,7 +80,13 @@ const SearchSubscriptionToggle: React.FC = ({ > Follow - setButtonEl(null)} /> + setButtonEl(null)} + /> ) } diff --git a/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx b/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx index 64487d1b25..03100dabeb 100644 --- a/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx +++ b/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx @@ -11,10 +11,12 @@ const StyledPopover = styled(Popover)({ const HeaderText = styled(Typography)(({ theme }) => ({ color: theme.custom.colors.darkGray2, marginBottom: "8px", + ...theme.typography.subtitle2, })) const BodyText = styled(Typography)(({ theme }) => ({ color: theme.custom.colors.silverGrayDark, marginBottom: "16px", + ...theme.typography.body2, })) const Footer = styled.div({ diff --git a/frontends/ol-components/src/components/Button/Button.tsx b/frontends/ol-components/src/components/Button/Button.tsx index ac88f0db5d..641e264f18 100644 --- a/frontends/ol-components/src/components/Button/Button.tsx +++ b/frontends/ol-components/src/components/Button/Button.tsx @@ -12,6 +12,7 @@ type ButtonVariant = | "text" | "noBorder" | "inverted" + | "success" type ButtonSize = "small" | "medium" | "large" type ButtonEdge = "circular" | "rounded" | "none" @@ -127,6 +128,27 @@ const ButtonStyled = styled.button((props) => { borderColor: "currentcolor", borderStyle: "solid", }, + variant === "success" && { + backgroundColor: colors.darkGreen, + color: colors.white, + border: "none", + /* Shadow/04dp */ + boxShadow: + "0px 2px 4px 0px rgba(37, 38, 43, 0.10), 0px 3px 8px 0px rgba(37, 38, 43, 0.12)", + ":hover:not(:disabled)": { + backgroundColor: colors.darkGreen, + boxShadow: "none", + }, + ":disabled": { + backgroundColor: colors.silverGray, + boxShadow: "none", + }, + }, + hasBorder && { + backgroundColor: "transparent", + borderColor: "currentcolor", + borderStyle: "solid", + }, variant === "secondary" && { color: colors.red, ":hover:not(:disabled)": { diff --git a/frontends/ol-components/src/components/Popover/Popover.tsx b/frontends/ol-components/src/components/Popover/Popover.tsx index 48a332c227..39e96ff8af 100644 --- a/frontends/ol-components/src/components/Popover/Popover.tsx +++ b/frontends/ol-components/src/components/Popover/Popover.tsx @@ -128,6 +128,7 @@ const Arrow = styled("div")({ const Content = styled.div(({ theme }) => ({ padding: "16px", backgroundColor: theme.custom.colors.white, + borderRadius: "8px", boxShadow: "0px 2px 4px 0px rgba(37, 38, 43, 0.10), 0px 6px 24px 0px rgba(37, 38, 43, 0.24)", })) From 9e9fc0745b287058f06963d8324bd45935cdf19b Mon Sep 17 00:00:00 2001 From: Anastasia Beglova Date: Mon, 23 Sep 2024 12:18:31 -0400 Subject: [PATCH 05/13] Make search mode defaults settable env variables (#1590) --- app.json | 20 +++++++ frontends/mit-learn/jest.config.ts | 5 ++ .../SearchDisplay/SearchDisplay.tsx | 29 +++++++--- .../src/pages/SearchPage/SearchPage.test.tsx | 39 ++++++++----- frontends/mit-learn/webpack.config.js | 30 ++++++++++ .../ol-utilities/src/types/settings.d.ts | 5 ++ learning_resources_search/api.py | 30 +++++++++- learning_resources_search/api_test.py | 58 ++++++++++++++++++- learning_resources_search/serializers.py | 3 - main/settings.py | 13 +++++ openapi/specs/v1.yaml | 15 ----- 11 files changed, 204 insertions(+), 43 deletions(-) diff --git a/app.json b/app.json index e47e0e6f8e..4fe3feccae 100644 --- a/app.json +++ b/app.json @@ -71,6 +71,26 @@ "description": "The domain to set the CSRF cookie on", "required": false }, + "DEFAULT_SEARCH_MODE": { + "description": "Default search mode for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_SLOP": { + "description": "Default slop value for the search API and frontend. Only used for phrase queries.", + "required": false + }, + "DEFAULT_SEARCH_STALENESS_PENALTY": { + "description": "Default staleness penalty value for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF": { + "description": "Default minimum score cutoff value for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY": { + "description": "Default max incompleteness penalty value for the search API and frontend", + "required": false + }, "EDX_API_ACCESS_TOKEN_URL": { "description": "URL to retrieve a MITx access token", "required": false diff --git a/frontends/mit-learn/jest.config.ts b/frontends/mit-learn/jest.config.ts index c767e8583f..54925827a7 100644 --- a/frontends/mit-learn/jest.config.ts +++ b/frontends/mit-learn/jest.config.ts @@ -19,6 +19,11 @@ const config: Config.InitialOptions = { MITOL_API_BASE_URL: "https://api.test.learn.mit.edu", PUBLIC_URL: "", SITE_NAME: "MIT Learn", + DEFAULT_SEARCH_MODE: "phrase", + DEFAULT_SEARCH_SLOP: 6, + DEFAULT_SEARCH_STALENESS_PENALTY: 2.5, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: 0, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: 90, }, }, } diff --git a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx index d269fa71ae..7c27760057 100644 --- a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -172,6 +172,11 @@ const FacetStyles = styled.div` &.facets-expanded { max-height: 600px; + + &.admin-facet { + max-height: fit-content; + } + transition: max-height 0.4s ease-in; } @@ -515,6 +520,14 @@ interface SearchDisplayProps { filterHeadingEl: React.ElementType } +const { + DEFAULT_SEARCH_MODE, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, +} = APP_SETTINGS + const SearchDisplay: React.FC = ({ page, setPage, @@ -585,7 +598,7 @@ const SearchDisplay: React.FC = ({ const searchModeDropdown = ( setSearchParams((prev) => { const next = new URLSearchParams(prev) @@ -623,7 +636,7 @@ const SearchDisplay: React.FC = ({ return (
- {searchParams.get("search_mode") === "phrase" ? ( + {(!searchParams.get("search_mode") && + DEFAULT_SEARCH_MODE === "phrase") || + searchParams.get("search_mode") === "phrase" ? (
Slop @@ -674,7 +689,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("slop") ? Number(searchParams.get("slop")) - : 0 + : DEFAULT_SEARCH_SLOP } setSearchParams={setSearchParams} urlParam="slop" @@ -694,7 +709,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("min_score") ? Number(searchParams.get("min_score")) - : 0 + : DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF } setSearchParams={setSearchParams} urlParam="min_score" @@ -713,7 +728,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("max_incompleteness_penalty") ? Number(searchParams.get("max_incompleteness_penalty")) - : 0 + : DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY } setSearchParams={setSearchParams} urlParam="max_incompleteness_penalty" diff --git a/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx b/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx index 4a5fae0258..697d3f9ae5 100644 --- a/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx +++ b/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx @@ -306,6 +306,11 @@ describe("SearchPage", () => { setMockResponse.get(urls.userMe.get(), { is_learning_path_editor: true, }) + APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" + APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 + APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 + APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 renderWithProviders() await waitFor(() => { const adminFacetContainer = screen.getByText("Admin Options") @@ -321,6 +326,11 @@ describe("SearchPage", () => { }) test("admin users can set the search mode and slop", async () => { + APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" + APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 + APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 + APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 setMockApiResponses({ search: { count: 700, @@ -349,14 +359,24 @@ test("admin users can set the search mode and slop", async () => { user.click(adminFacetContainer) }) - let slopSlider = screen.queryByText("Slop") - expect(slopSlider).toBeNull() - - const searchModeDropdowns = await screen.findAllByText("best_fields") + const searchModeDropdowns = await screen.findAllByText("phrase") const searchModeDropdown = searchModeDropdowns[0] await user.click(searchModeDropdown) + const mostFieldsSelect = await screen.findByRole("option", { + name: "most_fields", + }) + + await user.click(mostFieldsSelect) + + expect(location.current.search).toBe("?search_mode=most_fields") + + const slopSlider = screen.queryByText("Slop") + expect(slopSlider).toBeNull() + + await user.click(searchModeDropdown) + const phraseSelect = await screen.findByRole("option", { name: "phrase", }) @@ -370,17 +390,6 @@ test("admin users can set the search mode and slop", async () => { }) await user.click(searchModeDropdown) - - const mostFieldsSelect = await screen.findByRole("option", { - name: "most_fields", - }) - - await user.click(mostFieldsSelect) - - expect(location.current.search).toBe("?search_mode=most_fields") - - slopSlider = screen.queryByText("Slop") - expect(slopSlider).toBeNull() }) describe("Search Page Tabs", () => { diff --git a/frontends/mit-learn/webpack.config.js b/frontends/mit-learn/webpack.config.js index 7e49fd3bbb..9d48b30bd0 100644 --- a/frontends/mit-learn/webpack.config.js +++ b/frontends/mit-learn/webpack.config.js @@ -45,6 +45,11 @@ const { CSRF_COOKIE_NAME, APPZI_URL, MITOL_NOINDEX, + DEFAULT_SEARCH_MODE, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, } = cleanEnv(process.env, { NODE_ENV: str({ choices: ["development", "production", "test"], @@ -124,6 +129,26 @@ const { desc: "Whether to include a noindex meta tag", default: true, }), + DEFAULT_SEARCH_SLOP: num({ + desc: "The default search slop", + default: 6, + }), + DEFAULT_SEARCH_STALENESS_PENALTY: num({ + desc: "The default search staleness penalty", + default: 2.5, + }), + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: num({ + desc: "The default search minimum score cutoff", + default: 0, + }), + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: num({ + desc: "The default search max incompleteness penalty", + default: 90, + }), + DEFAULT_SEARCH_MODE: str({ + desc: "The default search mode", + default: "phrase", + }), }) const MITOL_FEATURES_PREFIX = "FEATURE_" @@ -265,6 +290,11 @@ module.exports = (env, argv) => { MITOL_SUPPORT_EMAIL: JSON.stringify(MITOL_SUPPORT_EMAIL), PUBLIC_URL: JSON.stringify(PUBLIC_URL), CSRF_COOKIE_NAME: JSON.stringify(CSRF_COOKIE_NAME), + DEFAULT_SEARCH_MODE: JSON.stringify(DEFAULT_SEARCH_MODE), + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, }, }), ] diff --git a/frontends/ol-utilities/src/types/settings.d.ts b/frontends/ol-utilities/src/types/settings.d.ts index da51840dd7..553c75e0c8 100644 --- a/frontends/ol-utilities/src/types/settings.d.ts +++ b/frontends/ol-utilities/src/types/settings.d.ts @@ -22,5 +22,10 @@ export declare global { SITE_NAME: string MITOL_SUPPORT_EMAIL: string PUBLIC_URL: string + DEFAULT_SEARCH_MODE: string + DEFAULT_SEARCH_SLOP: number + DEFAULT_SEARCH_STALENESS_PENALTY: number + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: number + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: number } } diff --git a/learning_resources_search/api.py b/learning_resources_search/api.py index eb4582a581..f6e08a07d1 100644 --- a/learning_resources_search/api.py +++ b/learning_resources_search/api.py @@ -5,6 +5,7 @@ from collections import Counter from datetime import UTC, datetime +from django.conf import settings from opensearch_dsl import Search from opensearch_dsl.query import MoreLikeThis, Percolate from opensearchpy.exceptions import NotFoundError @@ -507,7 +508,19 @@ def adjust_original_query_for_percolate(query): Remove keys that are irrelevent when storing original queries for percolate uniqueness such as "limit" and "offset" """ - for key in ["limit", "offset", "sortby", "yearly_decay_percent", "dev_mode"]: + for key in [ + "limit", + "offset", + "sortby", + "yearly_decay_percent", + "dev_mode", + "use_dfs_query_then_fetch", + "max_incompleteness_penalty", + "min_score", + "search_mode", + "slop", + "use_dfs_query_then_fetch", + ]: query.pop(key, None) return order_params(query) @@ -684,6 +697,21 @@ def execute_learn_search(search_params): Returns: dict: The opensearch response dict """ + if search_params.get("endpoint") != CONTENT_FILE_TYPE: + if search_params.get("yearly_decay_percent") is None: + search_params["yearly_decay_percent"] = ( + settings.DEFAULT_SEARCH_STALENESS_PENALTY + ) + if search_params.get("search_mode") is None: + search_params["search_mode"] = settings.DEFAULT_SEARCH_MODE + if search_params.get("slop") is None: + search_params["slop"] = settings.DEFAULT_SEARCH_SLOP + if search_params.get("min_score") is None: + search_params["min_score"] = settings.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF + if search_params.get("max_incompleteness_penalty") is None: + search_params["max_incompleteness_penalty"] = ( + settings.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY + ) search = construct_search(search_params) return search.execute().to_dict() diff --git a/learning_resources_search/api_test.py b/learning_resources_search/api_test.py index 71282429f5..86b810113b 100644 --- a/learning_resources_search/api_test.py +++ b/learning_resources_search/api_test.py @@ -1051,6 +1051,10 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "offset": 1, "sortby": "-readable_id", "endpoint": LEARNING_RESOURCE, + "yearly_decay_percent": 0, + "max_incompleteness_penalty": 0, + "min_score": 0, + "search_mode": "best_fields", } query = { @@ -1079,6 +1083,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1090,6 +1095,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "topics.name" ], + "type": "best_fields", } }, } @@ -1104,6 +1110,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1117,6 +1124,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1132,6 +1140,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -1150,6 +1159,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -1168,6 +1178,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -1194,6 +1205,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1203,6 +1215,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "best_fields", } }, } @@ -1217,6 +1230,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1230,6 +1244,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1245,6 +1260,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -1263,6 +1279,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -1281,6 +1298,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -1450,8 +1468,12 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): ], ) def test_execute_learn_search_with_script_score( - mocker, opensearch, yearly_decay_percent, max_incompleteness_penalty + mocker, settings, opensearch, yearly_decay_percent, max_incompleteness_penalty ): + settings.DEFAULT_SEARCH_MODE = "phrase" + settings.DEFAULT_SEARCH_SLOP = 0 + settings.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + opensearch.conn.search.return_value = { "hits": {"total": {"value": 10, "relation": "eq"}} } @@ -1531,6 +1553,7 @@ def test_execute_learn_search_with_script_score( "course_feature", "video.transcript.english", ], + "type": "phrase", } }, { @@ -1542,6 +1565,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "topics.name" ], + "type": "phrase", } }, } @@ -1556,6 +1580,7 @@ def test_execute_learn_search_with_script_score( "departments.department_id", "departments.name", ], + "type": "phrase", } }, } @@ -1569,6 +1594,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "course.course_numbers.value" ], + "type": "phrase", } }, } @@ -1584,6 +1610,7 @@ def test_execute_learn_search_with_script_score( "runs.semester", "runs.level", ], + "type": "phrase", } }, } @@ -1602,6 +1629,7 @@ def test_execute_learn_search_with_script_score( "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "phrase", } }, } @@ -1620,6 +1648,7 @@ def test_execute_learn_search_with_script_score( "short_description.english^2", "content_feature_type", ], + "type": "phrase", } }, "score_mode": "avg", @@ -1646,6 +1675,7 @@ def test_execute_learn_search_with_script_score( "course_feature", "video.transcript.english", ], + "type": "phrase", } }, { @@ -1655,6 +1685,7 @@ def test_execute_learn_search_with_script_score( "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "phrase", } }, } @@ -1669,6 +1700,7 @@ def test_execute_learn_search_with_script_score( "departments.department_id", "departments.name", ], + "type": "phrase", } }, } @@ -1682,6 +1714,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "course.course_numbers.value" ], + "type": "phrase", } }, } @@ -1697,6 +1730,7 @@ def test_execute_learn_search_with_script_score( "runs.semester", "runs.level", ], + "type": "phrase", } }, } @@ -1715,6 +1749,7 @@ def test_execute_learn_search_with_script_score( "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "phrase", } }, } @@ -1733,6 +1768,7 @@ def test_execute_learn_search_with_script_score( "short_description.english^2", "content_feature_type", ], + "type": "phrase", } }, "score_mode": "avg", @@ -1898,11 +1934,15 @@ def test_execute_learn_search_with_script_score( ) -def test_execute_learn_search_with_min_score(mocker, opensearch): +def test_execute_learn_search_with_min_score(mocker, settings, opensearch): opensearch.conn.search.return_value = { "hits": {"total": {"value": 10, "relation": "eq"}} } + settings.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 0 + settings.DEFAULT_SEARCH_STALENESS_PENALTY = 0 + settings.DEFAULT_SEARCH_MODE = "best_fields" + search_params = { "aggregations": ["offered_by"], "q": "math", @@ -1943,6 +1983,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1954,6 +1995,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "topics.name" ], + "type": "best_fields", } }, } @@ -1968,6 +2010,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1981,6 +2024,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1996,6 +2040,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -2014,6 +2059,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -2032,6 +2078,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -2058,6 +2105,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -2067,6 +2115,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "best_fields", } }, } @@ -2081,6 +2130,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -2094,6 +2144,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -2109,6 +2160,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -2127,6 +2179,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -2145,6 +2198,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", diff --git a/learning_resources_search/serializers.py b/learning_resources_search/serializers.py index 54225e29fa..fd9bd3fdcf 100644 --- a/learning_resources_search/serializers.py +++ b/learning_resources_search/serializers.py @@ -331,7 +331,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=2.5, help_text=( "Relevance score penalty percent per year for for resources without " "upcoming runs. Only affects results if there is a search term." @@ -425,7 +424,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=0, help_text=( "Minimum score value a text query result needs to have to be displayed" ), @@ -435,7 +433,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=0, help_text=( "Maximum score penalty for incomplete OCW courses in percent. " "An OCW course with completeness = 0 will have this score penalty. " diff --git a/main/settings.py b/main/settings.py index 3b3adc4d56..38d644143c 100644 --- a/main/settings.py +++ b/main/settings.py @@ -766,3 +766,16 @@ def get_all_config_keys(): # Enable or disable search engine indexing MITOL_NOINDEX = get_bool("MITOL_NOINDEX", True) # noqa: FBT003 + +# Search defaults settings - adjustable throught the admin ui +DEFAULT_SEARCH_MODE = get_string(name="DEFAULT_SEARCH_MODE", default="phrase") +DEFAULT_SEARCH_SLOP = get_int(name="DEFAULT_SEARCH_SLOP", default=6) +DEFAULT_SEARCH_STALENESS_PENALTY = get_float( + name="DEFAULT_SEARCH_STALENESS_PENALTY", default=2.5 +) +DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = get_float( + name="DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF", default=0 +) +DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = get_float( + name="DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY", default=90 +) diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index af4d7a9bbf..ea686f47a4 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -2493,7 +2493,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -2506,7 +2505,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -2717,7 +2715,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -2976,7 +2973,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -2989,7 +2985,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -3200,7 +3195,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -3484,7 +3478,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -3497,7 +3490,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -3722,7 +3714,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -3983,7 +3974,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -3996,7 +3986,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -4221,7 +4210,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -10055,7 +10043,6 @@ components: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. certification: @@ -10136,7 +10123,6 @@ components: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed max_incompleteness_penalty: @@ -10145,7 +10131,6 @@ components: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. From ac06a68cf8fc5a015b6d4e188254f1c52d830253 Mon Sep 17 00:00:00 2001 From: Doof Date: Mon, 23 Sep 2024 16:20:38 +0000 Subject: [PATCH 06/13] Release date for 0.19.2 --- RELEASE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index e8739530e8..7655b99689 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,7 +1,7 @@ Release Notes ============= -Version 0.19.2 +Version 0.19.2 (Released September 23, 2024) -------------- - Codespace opensearch service fix (#1582) From dcb1a9ac65e529df0dbc6fb7350d9c4db03ab076 Mon Sep 17 00:00:00 2001 From: Doof Date: Mon, 23 Sep 2024 16:21:13 +0000 Subject: [PATCH 07/13] Release 0.19.3 --- RELEASE.rst | 7 +++++++ main/settings.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index 7655b99689..1f970b3ec9 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,13 @@ Release Notes ============= +Version 0.19.3 +-------------- + +- Make search mode defaults settable env variables (#1590) +- follow/unfollow popover (#1589) +- Fix extract_openedx_data and backpopulate_mit_edx_data commands to work with course/program datafiles (#1587) + Version 0.19.2 (Released September 23, 2024) -------------- diff --git a/main/settings.py b/main/settings.py index f73c0069d9..c4e3864445 100644 --- a/main/settings.py +++ b/main/settings.py @@ -33,7 +33,7 @@ from main.settings_pluggy import * # noqa: F403 from openapi.settings_spectacular import open_spectacular_settings -VERSION = "0.19.2" +VERSION = "0.19.3" log = logging.getLogger() From 9d67642926c566b55aab9969c93e04f931e65349 Mon Sep 17 00:00:00 2001 From: Doof Date: Mon, 23 Sep 2024 19:22:18 +0000 Subject: [PATCH 08/13] Release date for 0.19.3 --- RELEASE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index 1f970b3ec9..1ae708ba43 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,7 +1,7 @@ Release Notes ============= -Version 0.19.3 +Version 0.19.3 (Released September 23, 2024) -------------- - Make search mode defaults settable env variables (#1590) From 4bde7f1742afc38406581789287d759c9a918285 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Tue, 24 Sep 2024 08:48:46 -0400 Subject: [PATCH 09/13] Pace and format fields for learning resources (#1588) --- .pre-commit-config.yaml | 2 +- frontends/api/src/generated/v1/api.ts | 152 +++++++++++ .../test-utils/factories/learningResources.ts | 18 ++ learning_resources/constants.py | 14 + learning_resources/etl/constants.py | 2 +- learning_resources/etl/micromasters.py | 78 ++++-- learning_resources/etl/micromasters_test.py | 9 + learning_resources/etl/mitxonline.py | 52 +++- learning_resources/etl/mitxonline_test.py | 18 +- learning_resources/etl/ocw.py | 6 + learning_resources/etl/ocw_test.py | 6 + learning_resources/etl/oll.py | 6 + learning_resources/etl/oll_test.py | 8 + learning_resources/etl/openedx.py | 50 +++- learning_resources/etl/openedx_test.py | 16 ++ learning_resources/etl/prolearn.py | 10 +- learning_resources/etl/prolearn_test.py | 12 + learning_resources/etl/sloan.py | 54 +++- learning_resources/etl/sloan_test.py | 57 ++++ learning_resources/etl/xpro.py | 10 + learning_resources/etl/xpro_test.py | 14 + .../0068_learningresource_format_pace.py | 75 +++++ learning_resources/models.py | 40 +++ learning_resources/serializers.py | 36 +++ learning_resources/serializers_test.py | 9 + learning_resources_search/constants.py | 28 ++ main/models.py | 2 +- openapi/specs/v1.yaml | 256 ++++++++++++++++++ 28 files changed, 993 insertions(+), 47 deletions(-) create mode 100644 learning_resources/migrations/0068_learningresource_format_pace.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17d9ff63e6..0a30546482 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,7 +74,7 @@ repos: - ".*/generated/" additional_dependencies: ["gibberish-detector"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.6.5" + rev: "v0.6.6" hooks: - id: ruff-format - id: ruff diff --git a/frontends/api/src/generated/v1/api.ts b/frontends/api/src/generated/v1/api.ts index 983db187e9..ff24f6357f 100644 --- a/frontends/api/src/generated/v1/api.ts +++ b/frontends/api/src/generated/v1/api.ts @@ -686,6 +686,18 @@ export interface CourseResource { * @memberof CourseResource */ resource_category: string + /** + * + * @type {Array} + * @memberof CourseResource + */ + format: Array + /** + * + * @type {Array} + * @memberof CourseResource + */ + pace: Array /** * * @type {CourseResourceResourceTypeEnum} @@ -844,6 +856,62 @@ export const CourseResourceDeliveryInnerCodeEnum = { export type CourseResourceDeliveryInnerCodeEnum = (typeof CourseResourceDeliveryInnerCodeEnum)[keyof typeof CourseResourceDeliveryInnerCodeEnum] +/** + * + * @export + * @interface CourseResourceFormatInner + */ +export interface CourseResourceFormatInner { + /** + * + * @type {string} + * @memberof CourseResourceFormatInner + */ + code: CourseResourceFormatInnerCodeEnum + /** + * + * @type {string} + * @memberof CourseResourceFormatInner + */ + name: string +} + +export const CourseResourceFormatInnerCodeEnum = { + Synchronous: "synchronous", + Asynchronous: "asynchronous", +} as const + +export type CourseResourceFormatInnerCodeEnum = + (typeof CourseResourceFormatInnerCodeEnum)[keyof typeof CourseResourceFormatInnerCodeEnum] + +/** + * + * @export + * @interface CourseResourcePaceInner + */ +export interface CourseResourcePaceInner { + /** + * + * @type {string} + * @memberof CourseResourcePaceInner + */ + code: CourseResourcePaceInnerCodeEnum + /** + * + * @type {string} + * @memberof CourseResourcePaceInner + */ + name: string +} + +export const CourseResourcePaceInnerCodeEnum = { + SelfPaced: "self_paced", + InstructorPaced: "instructor_paced", +} as const + +export type CourseResourcePaceInnerCodeEnum = + (typeof CourseResourcePaceInnerCodeEnum)[keyof typeof CourseResourcePaceInnerCodeEnum] + /** * Serializer for course resources * @export @@ -1376,6 +1444,18 @@ export interface LearningPathResource { * @memberof LearningPathResource */ resource_category: string + /** + * + * @type {Array} + * @memberof LearningPathResource + */ + format: Array + /** + * + * @type {Array} + * @memberof LearningPathResource + */ + pace: Array /** * * @type {LearningPathResourceResourceTypeEnum} @@ -2100,6 +2180,18 @@ export interface LearningResourceRun { * @memberof LearningResourceRun */ delivery: Array + /** + * + * @type {Array} + * @memberof LearningResourceRun + */ + format: Array + /** + * + * @type {Array} + * @memberof LearningResourceRun + */ + pace: Array /** * * @type {string} @@ -4049,6 +4141,18 @@ export interface PodcastEpisodeResource { * @memberof PodcastEpisodeResource */ resource_category: string + /** + * + * @type {Array} + * @memberof PodcastEpisodeResource + */ + format: Array + /** + * + * @type {Array} + * @memberof PodcastEpisodeResource + */ + pace: Array /** * * @type {PodcastEpisodeResourceResourceTypeEnum} @@ -4395,6 +4499,18 @@ export interface PodcastResource { * @memberof PodcastResource */ resource_category: string + /** + * + * @type {Array} + * @memberof PodcastResource + */ + format: Array + /** + * + * @type {Array} + * @memberof PodcastResource + */ + pace: Array /** * * @type {PodcastResourceResourceTypeEnum} @@ -4961,6 +5077,18 @@ export interface ProgramResource { * @memberof ProgramResource */ resource_category: string + /** + * + * @type {Array} + * @memberof ProgramResource + */ + format: Array + /** + * + * @type {Array} + * @memberof ProgramResource + */ + pace: Array /** * * @type {ProgramResourceResourceTypeEnum} @@ -5786,6 +5914,18 @@ export interface VideoPlaylistResource { * @memberof VideoPlaylistResource */ resource_category: string + /** + * + * @type {Array} + * @memberof VideoPlaylistResource + */ + format: Array + /** + * + * @type {Array} + * @memberof VideoPlaylistResource + */ + pace: Array /** * * @type {VideoPlaylistResourceResourceTypeEnum} @@ -6120,6 +6260,18 @@ export interface VideoResource { * @memberof VideoResource */ resource_category: string + /** + * + * @type {Array} + * @memberof VideoResource + */ + format: Array + /** + * + * @type {Array} + * @memberof VideoResource + */ + pace: Array /** * * @type {VideoResourceResourceTypeEnum} diff --git a/frontends/api/src/test-utils/factories/learningResources.ts b/frontends/api/src/test-utils/factories/learningResources.ts index 8c11ddf195..f37c0092fb 100644 --- a/frontends/api/src/test-utils/factories/learningResources.ts +++ b/frontends/api/src/test-utils/factories/learningResources.ts @@ -30,6 +30,8 @@ import type { import { AvailabilityEnum, DeliveryEnum, + CourseResourcePaceInnerCodeEnum, + CourseResourceFormatInnerCodeEnum, ResourceTypeEnum, LearningResourceRunLevelInnerCodeEnum, PlatformEnum, @@ -186,6 +188,22 @@ const learningResourceRun: Factory = (overrides = {}) => { name: uniqueEnforcerWords.enforce(() => faker.lorem.words()), }, ], + pace: [ + { + code: faker.helpers.arrayElement( + Object.values(CourseResourcePaceInnerCodeEnum), + ), + name: uniqueEnforcerWords.enforce(() => faker.lorem.words()), + }, + ], + format: [ + { + code: faker.helpers.arrayElement( + Object.values(CourseResourceFormatInnerCodeEnum), + ), + name: uniqueEnforcerWords.enforce(() => faker.lorem.words()), + }, + ], level: [ { code: faker.helpers.arrayElement( diff --git a/learning_resources/constants.py b/learning_resources/constants.py index cdc609d824..41e84d6edb 100644 --- a/learning_resources/constants.py +++ b/learning_resources/constants.py @@ -287,3 +287,17 @@ class CertificationType(ExtendedEnum): professional = "Professional Certificate" completion = "Certificate of Completion" none = "No Certificate" + + +class Pace(ExtendedEnum): + """Enum for resource pace types""" + + self_paced = "Self-paced" + instructor_paced = "Instructor-paced" + + +class Format(ExtendedEnum): + """Enum for resource format types""" + + synchronous = "Synchronous" + asynchronous = "Asynchronous" diff --git a/learning_resources/etl/constants.py b/learning_resources/etl/constants.py index 319f85f245..fd1e7fcc6a 100644 --- a/learning_resources/etl/constants.py +++ b/learning_resources/etl/constants.py @@ -46,7 +46,7 @@ ) -class ETLSource(Enum): +class ETLSource(ExtendedEnum): """Enum of ETL sources""" micromasters = "micromasters" diff --git a/learning_resources/etl/micromasters.py b/learning_resources/etl/micromasters.py index b4bec1232e..6bebb30d35 100644 --- a/learning_resources/etl/micromasters.py +++ b/learning_resources/etl/micromasters.py @@ -8,12 +8,13 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, OfferedBy, PlatformType, ) from learning_resources.etl.constants import COMMON_HEADERS, ETLSource -from learning_resources.models import LearningResource +from learning_resources.models import LearningResource, default_pace OFFERED_BY = {"code": OfferedBy.mitx.name} READABLE_ID_PREFIX = "micromasters-program-" @@ -46,6 +47,27 @@ def _is_published(course_id: str) -> bool: return False +def _get_course_pace(course_id: str) -> list[str]: + """ + Determine the pace of the course by id + + Args: + course_id (str): the course id + + Returns: + list[str]: the pace of the course as a list of strings + + """ + existing_course = LearningResource.objects.filter( + readable_id=course_id, + resource_type=LearningResourceType.course.name, + published=True, + ).first() + if existing_course: + return existing_course.pace + return default_pace() + + def _transform_image(micromasters_data: dict) -> dict: """ Transform an image into our normalized data structure @@ -65,7 +87,36 @@ def transform(programs_data): programs = [] for program in programs_data: url = program.get("programpage_url") + # need positioning of courses by course_id for course data + courses = [ + { + "readable_id": course["edx_key"], + "platform": PlatformType.edx.name, + "offered_by": OFFERED_BY, + "published": _is_published(course["edx_key"]), + "pace": _get_course_pace(course["edx_key"]), + "runs": [ + { + "run_id": run["edx_course_key"], + } + for run in course["course_runs"] + if run.get("edx_course_key", None) + ], + } + for course in sorted( + program["courses"], + key=lambda course: course["position_in_program"], + ) + ] if url and DEDP not in url: + pace = sorted( + { + pace + for course in courses + for pace in course["pace"] + if course["published"] + } + ) programs.append( { "readable_id": f"{READABLE_ID_PREFIX}{program['id']}", @@ -91,30 +142,15 @@ def transform(programs_data): "end_date": program["end_date"], "enrollment_start": program["enrollment_start"], "availability": Availability.dated.name, + "pace": pace, + "format": [Format.asynchronous.name], } ], "topics": program["topics"], "availability": Availability.dated.name, - # only need positioning of courses by course_id for course data - "courses": [ - { - "readable_id": course["edx_key"], - "platform": PlatformType.edx.name, - "offered_by": OFFERED_BY, - "published": _is_published(course["edx_key"]), - "runs": [ - { - "run_id": run["edx_course_key"], - } - for run in course["course_runs"] - if run.get("edx_course_key", None) - ], - } - for course in sorted( - program["courses"], - key=lambda course: course["position_in_program"], - ) - ], + "pace": pace, + "format": [Format.asynchronous.name], + "courses": courses, } ) return programs diff --git a/learning_resources/etl/micromasters_test.py b/learning_resources/etl/micromasters_test.py index ede2c8df77..343d61260c 100644 --- a/learning_resources/etl/micromasters_test.py +++ b/learning_resources/etl/micromasters_test.py @@ -6,7 +6,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, + Pace, PlatformType, ) from learning_resources.etl import micromasters @@ -111,6 +113,7 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): readable_id="1", resource_type=LearningResourceType.course.name, etl_source=ETLSource.mit_edx.name, + pace=[Pace.instructor_paced.name], ) if missing_url: mock_micromasters_data[0]["programpage_url"] = None @@ -129,6 +132,8 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "certification": True, "certification_type": CertificationType.micromasters.name, "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], "courses": [ { "readable_id": "1", @@ -140,6 +145,7 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "run_id": "course_key_1", } ], + "pace": [Pace.instructor_paced.name], }, { "readable_id": "2", @@ -147,6 +153,7 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "offered_by": micromasters.OFFERED_BY, "published": False, "runs": [], + "pace": [Pace.self_paced.name], }, ], "runs": [ @@ -162,6 +169,8 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "end_date": None, "enrollment_start": "2019-09-29T20:13:26.367297Z", "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } ], "topics": [{"name": "program"}, {"name": "first"}], diff --git a/learning_resources/etl/mitxonline.py b/learning_resources/etl/mitxonline.py index 7495bd9f12..296174d960 100644 --- a/learning_resources/etl/mitxonline.py +++ b/learning_resources/etl/mitxonline.py @@ -12,8 +12,10 @@ from learning_resources.constants import ( CertificationType, + Format, LearningResourceType, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -237,6 +239,12 @@ def _transform_run(course_run: dict, course: dict) -> dict: if parse_page_attribute(course, "page_url") else RunStatus.archived.value, "availability": course.get("availability"), + "format": [Format.asynchronous.name], + "pace": [ + Pace.self_paced.name + if course_run.get("is_self_paced", False) + else Pace.instructor_paced.name + ], } @@ -283,6 +291,8 @@ def _transform_course(course): "url": parse_page_attribute(course, "page_url", is_url=True), "description": clean_data(parse_page_attribute(course, "description")), "availability": course.get("availability"), + "format": [Format.asynchronous.name], + "pace": sorted({pace for run in runs for pace in run["pace"]}), } @@ -320,12 +330,30 @@ def _fetch_courses_by_ids(course_ids): return [] -def transform_programs(programs): - """Transform the MITX Online catalog data""" - # normalize the MITx Online data +def transform_programs(programs: list[dict]) -> list[dict]: + """ + Transform the MITX Online catalog data - return [ - { + Args: + programs (list of dict): the MITX Online programs data + + Returns: + list of dict: the transformed programs data + + """ + # normalize the MITx Online data + for program in programs: + courses = transform_courses( + [ + course + for course in _fetch_courses_by_ids(program["courses"]) + if not re.search(EXCLUDE_REGEX, course["title"], re.IGNORECASE) + ] + ) + pace = sorted( + {course_pace for course in courses for course_pace in course["pace"]} + ) + yield { "readable_id": program["readable_id"], "title": program["title"], "offered_by": OFFERED_BY, @@ -346,6 +374,8 @@ def transform_programs(programs): "published": bool( parse_page_attribute(program, "page_url") ), # a program is only considered published if it has a page url + "format": [Format.asynchronous.name], + "pace": pace, "runs": [ { "run_id": program["readable_id"], @@ -371,15 +401,9 @@ def transform_programs(programs): if parse_page_attribute(program, "page_url") else RunStatus.archived.value, "availability": program.get("availability"), + "format": [Format.asynchronous.name], + "pace": pace, } ], - "courses": transform_courses( - [ - course - for course in _fetch_courses_by_ids(program["courses"]) - if not re.search(EXCLUDE_REGEX, course["title"], re.IGNORECASE) - ] - ), + "courses": courses, } - for program in programs - ] diff --git a/learning_resources/etl/mitxonline_test.py b/learning_resources/etl/mitxonline_test.py index 637cc81209..a7364d393c 100644 --- a/learning_resources/etl/mitxonline_test.py +++ b/learning_resources/etl/mitxonline_test.py @@ -11,7 +11,9 @@ from learning_resources.constants import ( CertificationType, + Format, LearningResourceType, + Pace, PlatformType, RunStatus, ) @@ -150,6 +152,8 @@ def test_mitxonline_transform_programs( "url": parse_page_attribute(program_data, "page_url", is_url=True), "availability": program_data["availability"], "topics": transform_topics(program_data["topics"], OFFERED_BY["code"]), + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], "runs": [ { "run_id": program_data["readable_id"], @@ -171,6 +175,8 @@ def test_mitxonline_transform_programs( if parse_page_attribute(program_data, "page_url") else RunStatus.archived.value, "availability": program_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } ], "courses": [ @@ -205,6 +211,8 @@ def test_mitxonline_transform_programs( "certification_type": CertificationType.completion.name, "url": parse_page_attribute(course_data, "page_url", is_url=True), "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], "topics": transform_topics( course_data["topics"], OFFERED_BY["code"] ), @@ -250,6 +258,8 @@ def test_mitxonline_transform_programs( if parse_page_attribute(course_data, "page_url") else RunStatus.archived.value, "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } for course_run_data in course_data["courseruns"] ], @@ -379,6 +389,8 @@ def test_mitxonline_transform_courses(settings, mock_mitxonline_courses_data): if parse_page_attribute(course_data, "page_url") else RunStatus.archived.value, "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } for course_run_data in course_data["courseruns"] ], @@ -394,6 +406,8 @@ def test_mitxonline_transform_courses(settings, mock_mitxonline_courses_data): ] }, "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } for course_data in mock_mitxonline_courses_data["results"] if "PROCTORED EXAM" not in course_data["title"] @@ -463,7 +477,9 @@ def test_program_run_start_date_value( # noqa: PLR0913 mock_mitxonline_programs_data["results"][0]["start_date"] = start_dt mock_mitxonline_programs_data["results"][0]["enrollment_start"] = enrollment_dt - transformed_programs = transform_programs(mock_mitxonline_programs_data["results"]) + transformed_programs = list( + transform_programs(mock_mitxonline_programs_data["results"]) + ) assert transformed_programs[0]["runs"][0]["start_date"] == _parse_datetime( expected_dt diff --git a/learning_resources/etl/ocw.py b/learning_resources/etl/ocw.py index f11310b8e2..d776f63c82 100644 --- a/learning_resources/etl/ocw.py +++ b/learning_resources/etl/ocw.py @@ -20,9 +20,11 @@ CONTENT_TYPE_VIDEO, VALID_TEXT_FILE_TYPES, Availability, + Format, LearningResourceDelivery, LearningResourceType, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -285,6 +287,8 @@ def transform_run(course_data: dict) -> dict: "url": course_data["url"], "availability": Availability.anytime.name, "delivery": parse_delivery(course_data), + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } @@ -368,6 +372,8 @@ def transform_course(course_data: dict) -> dict: "availability": Availability.anytime.name, "delivery": parse_delivery(course_data), "license_cc": True, + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } diff --git a/learning_resources/etl/ocw_test.py b/learning_resources/etl/ocw_test.py index 5d2ba82828..d54b562ce7 100644 --- a/learning_resources/etl/ocw_test.py +++ b/learning_resources/etl/ocw_test.py @@ -12,7 +12,9 @@ from learning_resources.constants import ( DEPARTMENTS, Availability, + Format, LearningResourceDelivery, + Pace, ) from learning_resources.etl.constants import CourseNumberType, ETLSource from learning_resources.etl.ocw import ( @@ -248,6 +250,10 @@ def test_transform_course( # noqa: PLR0913 assert transformed_json["runs"][0]["delivery"] == expected_delivery assert transformed_json["runs"][0]["availability"] == Availability.anytime.name assert transformed_json["availability"] == Availability.anytime.name + assert transformed_json["runs"][0]["pace"] == [Pace.self_paced.name] + assert transformed_json["runs"][0]["format"] == [Format.asynchronous.name] + assert transformed_json["pace"] == [Pace.self_paced.name] + assert transformed_json["format"] == [Format.asynchronous.name] assert transformed_json["description"] == clean_data( course_json["course_description_html"] ) diff --git a/learning_resources/etl/oll.py b/learning_resources/etl/oll.py index 5c720cbafe..96bd8436b3 100644 --- a/learning_resources/etl/oll.py +++ b/learning_resources/etl/oll.py @@ -13,7 +13,9 @@ from learning_resources.constants import ( Availability, + Format, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -143,6 +145,8 @@ def transform_run(course_data: dict) -> list[dict]: ], "status": RunStatus.archived.value, "availability": Availability.anytime.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } ] @@ -183,6 +187,8 @@ def transform_course(course_data: dict) -> dict: "prices": [Decimal(0.00)], "etl_source": ETLSource.oll.name, "availability": Availability.anytime.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } diff --git a/learning_resources/etl/oll_test.py b/learning_resources/etl/oll_test.py index f0eebf8ea6..91e9e05c7d 100644 --- a/learning_resources/etl/oll_test.py +++ b/learning_resources/etl/oll_test.py @@ -73,6 +73,8 @@ def test_oll_transform(mocker, oll_course_data): "year": 2022, "status": "Archived", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } ], "image": { @@ -82,6 +84,8 @@ def test_oll_transform(mocker, oll_course_data): "prices": [0.00], "etl_source": "oll", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } assert results[2] == { "title": "Competency-Based Education", @@ -125,6 +129,8 @@ def test_oll_transform(mocker, oll_course_data): "year": 2019, "status": "Archived", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } ], "image": { @@ -134,4 +140,6 @@ def test_oll_transform(mocker, oll_course_data): "prices": [0.00], "etl_source": "oll", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } diff --git a/learning_resources/etl/openedx.py b/learning_resources/etl/openedx.py index c7cc98b532..18a1ce118c 100644 --- a/learning_resources/etl/openedx.py +++ b/learning_resources/etl/openedx.py @@ -18,7 +18,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, + Pace, PlatformType, RunStatus, ) @@ -290,7 +292,7 @@ def _parse_course_dates(program, date_field): [ run[date_field] for run in course["course_runs"] - if run["status"] == "published" and run[date_field] + if _get_run_published(run) and run[date_field] ] ) return dates @@ -312,7 +314,7 @@ def _get_course_price(course): for run in sorted( course["course_runs"], key=lambda x: x["start"], reverse=False ): - if run["status"] == "published": + if _get_run_published(run): return min( [ Decimal(seat["price"]) @@ -356,6 +358,8 @@ def _transform_course_run(config, course_run, course_last_modified, marketing_ur "image": _transform_course_image(course_run.get("image")), "status": course_run.get("availability"), "availability": _get_run_availability(course_run).name, + "format": [Format.asynchronous.name], + "pace": [course_run.get("pacing_type") or Pace.self_paced.name], "url": marketing_url or "{}{}/course/".format(config.alt_url, course_run.get("key")), "prices": sorted( @@ -401,6 +405,25 @@ def _parse_program_instructors_topics(program): ) +def _parse_course_pace(runs: list[dict]) -> list[str]: + """ + Parse the pace of a course based on its runs + + Args: + runs (list of dict): the runs data + + Returns: + str: the pace of the course or programe + """ + pace = sorted( + {run["pacing_type"] for run in runs if run and _get_run_published(run)} + ) + if len(pace) == 0: + # Archived courses are considered self-paced + pace = [Pace.self_paced.name] + return pace + + def _transform_program_course(config: OpenEdxConfiguration, course: dict) -> dict: """ Transform a program's course dict to a normalized data structure @@ -419,6 +442,7 @@ def _transform_program_course(config: OpenEdxConfiguration, course: dict) -> dic "platform": config.platform, "resource_type": LearningResourceType.course.name, "offered_by": {"code": config.offered_by}, + "pace": _parse_course_pace(course.get("course_runs", [])), } @@ -458,6 +482,14 @@ def _transform_program_run( "prices": [_sum_course_prices(program)], "instructors": program.pop("instructors", []), "availability": _get_program_availability(program), + "format": [Format.asynchronous.name], + "pace": sorted( + { + pace + for course in program.get("courses", []) + for pace in _parse_course_pace(course.get("course_runs", [])) + } + ), } @@ -507,6 +539,8 @@ def _transform_course(config: OpenEdxConfiguration, course: dict) -> dict: if has_certification else CertificationType.none.name, "availability": _get_course_availability(course), + "format": [Format.asynchronous.name], + "pace": _parse_course_pace(course.get("course_runs", [])), } @@ -526,6 +560,11 @@ def _transform_program(config: OpenEdxConfiguration, program: dict) -> dict: image = _transform_program_image(program) instructors, topics = _parse_program_instructors_topics(program) program["instructors"] = instructors + courses = [ + _transform_program_course(config, course) + for course in program.get("courses", []) + ] + paces = sorted({pace for course in courses for pace in course["pace"]}) runs = [_transform_program_run(program, last_modified, image)] has_certification = parse_certification(config.offered_by, runs) return { @@ -549,10 +588,9 @@ def _transform_program(config: OpenEdxConfiguration, program: dict) -> dict: if has_certification else CertificationType.none.name, "availability": runs[0]["availability"], - "courses": [ - _transform_program_course(config, course) - for course in program.get("courses", []) - ], + "format": [Format.asynchronous.name], + "pace": paces, + "courses": courses, } diff --git a/learning_resources/etl/openedx_test.py b/learning_resources/etl/openedx_test.py index 3503cc15c4..3ae662fbe7 100644 --- a/learning_resources/etl/openedx_test.py +++ b/learning_resources/etl/openedx_test.py @@ -10,8 +10,10 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -212,6 +214,13 @@ def test_transform_course( # noqa: PLR0913 "last_modified": any_instance_of(datetime), "topics": [{"name": "Data Analysis & Statistics"}], "url": "http://localhost/fake-alt-url/this_course", + "format": [Format.asynchronous.name], + "pace": [Pace.instructor_paced.name] + if has_runs + and not is_run_deleted + and is_run_published + and is_run_enrollable + else [Pace.self_paced.name], "published": is_run_published and is_run_enrollable and not is_run_deleted @@ -249,6 +258,8 @@ def test_transform_course( # noqa: PLR0913 "year": 2019, "published": is_run_enrollable and is_run_published, "availability": Availability.dated.name, + "format": [Format.asynchronous.name], + "pace": [Pace.instructor_paced.name], } ] ), @@ -422,6 +433,8 @@ def test_transform_program( "certification": False, "certification_type": CertificationType.none.name, "availability": Availability.anytime.name, + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], "runs": ( [ { @@ -450,6 +463,8 @@ def test_transform_program( "url": extracted[0]["marketing_url"], "published": True, "availability": Availability.anytime.name, + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } ] ), @@ -460,6 +475,7 @@ def test_transform_program( "platform": openedx_program_config.platform, "readable_id": f"MITx+6.002.{i}x", "resource_type": LearningResourceType.course.name, + "pace": [Pace.self_paced.name], } for i in range(1, 4) ], diff --git a/learning_resources/etl/prolearn.py b/learning_resources/etl/prolearn.py index de00d3b9d3..022aa6cc62 100644 --- a/learning_resources/etl/prolearn.py +++ b/learning_resources/etl/prolearn.py @@ -9,7 +9,7 @@ import requests from django.conf import settings -from learning_resources.constants import Availability, CertificationType +from learning_resources.constants import Availability, CertificationType, Format, Pace from learning_resources.etl.constants import ETLSource from learning_resources.etl.utils import transform_delivery, transform_topics from learning_resources.models import LearningResourceOfferor, LearningResourcePlatform @@ -281,10 +281,14 @@ def transform_programs(programs: list[dict]) -> list[dict]: } ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for course_id in sorted(program["field_related_courses_programs"]) ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } unique_program = unique_programs.setdefault( transformed_program["url"], transformed_program @@ -321,6 +325,8 @@ def _transform_runs(resource: dict) -> list[dict]: "url": parse_url(resource), "delivery": transform_delivery(resource["format_name"]), "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } ) return runs @@ -361,6 +367,8 @@ def _transform_course( "runs": runs, "unique_field": UNIQUE_FIELD, "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } return None diff --git a/learning_resources/etl/prolearn_test.py b/learning_resources/etl/prolearn_test.py index 00eb8fc267..9390b73a48 100644 --- a/learning_resources/etl/prolearn_test.py +++ b/learning_resources/etl/prolearn_test.py @@ -10,8 +10,10 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceDelivery, OfferedBy, + Pace, PlatformType, ) from learning_resources.etl.constants import ETLSource @@ -177,6 +179,8 @@ def test_prolearn_transform_programs(mock_csail_programs_data): ), "availability": Availability.dated.name, "delivery": transform_delivery(program["format_name"]), + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for (start_val, end_val) in zip( program["start_value"], program["end_value"] @@ -201,10 +205,14 @@ def test_prolearn_transform_programs(mock_csail_programs_data): } ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for course_id in sorted(program["field_related_courses_programs"]) ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for program in extracted_data[1:] ] @@ -254,6 +262,8 @@ def test_prolearn_transform_courses(mock_mitpe_courses_data): ), "availability": Availability.dated.name, "delivery": transform_delivery(course["format_name"]), + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for (start_val, end_val) in zip( course["start_value"], course["end_value"] @@ -261,6 +271,8 @@ def test_prolearn_transform_courses(mock_mitpe_courses_data): ], "course": {"course_numbers": []}, "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for course in extracted_data[2:] ] diff --git a/learning_resources/etl/sloan.py b/learning_resources/etl/sloan.py index 3697c5690d..a58b4e311e 100644 --- a/learning_resources/etl/sloan.py +++ b/learning_resources/etl/sloan.py @@ -12,7 +12,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -21,6 +23,7 @@ transform_delivery, transform_topics, ) +from learning_resources.models import default_format log = logging.getLogger(__name__) @@ -124,6 +127,49 @@ def parse_availability(run_data: dict) -> str: return Availability.dated.name +def parse_pace(run_data: dict) -> str: + """ + Parse pace from run data + + Args: + run_data (list): the run data + + Returns: + str: the pace + """ + if run_data and ( + run_data.get("Delivery") == "Online" + and run_data.get("Format") == "Asynchronous (On-Demand)" + ): + return Pace.self_paced.name + return Pace.instructor_paced.name + + +def parse_format(run_data: dict) -> str: + """ + Parse format from run data + + Args: + run_data (list): the run data + + Returns: + str: the format code + """ + if run_data: + delivery = run_data.get("Delivery") + if delivery == "In Person": + return [Format.synchronous.name] + elif delivery == "Blended": + return [Format.synchronous.name, Format.asynchronous.name] + else: + return ( + [Format.asynchronous.name] + if "Asynchronous" in (run_data.get("Format") or "") + else [Format.synchronous.name] + ) + return default_format() + + def extract(): """ Extract Sloan Executive Education data @@ -184,6 +230,8 @@ def transform_run(run_data, course_data): "published": True, "prices": [run_data["Price"]], "instructors": [{"full_name": name.strip()} for name in faculty_names], + "pace": [parse_pace(run_data)], + "format": parse_format(run_data), } @@ -205,6 +253,8 @@ def transform_course(course_data: dict, runs_data: dict) -> dict: format_delivery = list( {transform_delivery(run["Delivery"])[0] for run in course_runs_data} ) + runs = [transform_run(run, course_data) for run in course_runs_data] + transformed_course = { "readable_id": course_data["Course_Id"], "title": course_data["Title"], @@ -223,8 +273,10 @@ def transform_course(course_data: dict, runs_data: dict) -> dict: "course": { "course_numbers": [], }, - "runs": [transform_run(run, course_data) for run in course_runs_data], + "runs": runs, "continuing_ed_credits": course_runs_data[0]["Continuing_Ed_Credits"], + "pace": sorted({pace for run in runs for pace in run["pace"]}), + "format": sorted({run_format for run in runs for run_format in run["format"]}), } return transformed_course if transformed_course.get("url") else None diff --git a/learning_resources/etl/sloan_test.py b/learning_resources/etl/sloan_test.py index 9cb6a6cfa6..85c472d15e 100644 --- a/learning_resources/etl/sloan_test.py +++ b/learning_resources/etl/sloan_test.py @@ -7,13 +7,17 @@ from learning_resources.constants import ( Availability, + Format, + Pace, RunStatus, ) from learning_resources.etl.sloan import ( extract, parse_availability, parse_datetime, + parse_format, parse_image, + parse_pace, transform_course, transform_delivery, transform_run, @@ -124,6 +128,8 @@ def test_transform_run( "published": True, "prices": [run_data["Price"]], "instructors": [{"full_name": name.strip()} for name in faculty_names], + "pace": [Pace.instructor_paced.name], + "format": [Format.synchronous.name], } @@ -147,6 +153,10 @@ def test_transform_course(mock_sloan_courses_data, mock_sloan_runs_data): assert transformed["runs"][0]["availability"] == parse_availability( course_runs_data[0] ) + assert transformed["pace"] == [Pace.instructor_paced.name] + assert transformed["format"] == [Format.asynchronous.name, Format.synchronous.name] + assert transformed["runs"][0]["pace"] == [Pace.instructor_paced.name] + assert transformed["runs"][0]["format"] == [Format.asynchronous.name] assert transformed["image"] == parse_image(course_data) assert ( transformed["continuing_ed_credits"] @@ -171,6 +181,8 @@ def test_transform_course(mock_sloan_courses_data, mock_sloan_runs_data): "course", "runs", "continuing_ed_credits", + "pace", + "format", ] ) @@ -223,3 +235,48 @@ def test_enabled_flag(mock_sloan_api_setting, settings): """Extract should return empty lists if the SEE_API_ENABLED flag is False""" settings.SEE_API_ENABLED = False assert extract() == ([], []) + + +@pytest.mark.parametrize( + ("delivery", "run_format", "pace"), + [ + ("Online", "Synchronous", Pace.instructor_paced.name), + ("Online", "Asynchronous (On-Demand)", Pace.self_paced.name), + ("Online", "Asynchronous (Date based)", Pace.instructor_paced.name), + ("In Person", "Asynchronous (On-Demand)", Pace.instructor_paced.name), + ], +) +def test_parse_pace(delivery, run_format, pace): + """Test that the pace is parsed correctly""" + run_data = { + "Format": run_format, + "Delivery": delivery, + } + assert parse_pace(run_data) == pace + assert parse_pace(None) == Pace.instructor_paced.name + + +@pytest.mark.parametrize( + ("delivery", "run_format", "expected_format"), + [ + ("In Person", "Asynchronous (On-Demand)", [Format.synchronous.name]), + ( + "Blended", + "Asynchronous (On-Demand)", + [Format.synchronous.name, Format.asynchronous.name], + ), + ("Online", "Synchronous", [Format.synchronous.name]), + ("Online", "Asynchronous (On-Demand)", [Format.asynchronous.name]), + ("Online", "Asynchronous (Date based)", [Format.asynchronous.name]), + ("Online", None, [Format.synchronous.name]), + (None, None, [Format.synchronous.name]), + ], +) +def test_parse_format(delivery, run_format, expected_format): + """Test that the format is parsed correctly""" + run_data = { + "Format": run_format, + "Delivery": delivery, + } + assert parse_format(run_data) == expected_format + assert parse_format(None) == [Format.asynchronous.name] diff --git a/learning_resources/etl/xpro.py b/learning_resources/etl/xpro.py index 2dc7f08214..c6f3f4b225 100644 --- a/learning_resources/etl/xpro.py +++ b/learning_resources/etl/xpro.py @@ -10,8 +10,10 @@ from learning_resources.constants import ( CertificationType, + Format, LearningResourceType, OfferedBy, + Pace, PlatformType, ) from learning_resources.etl.constants import ETLSource @@ -117,6 +119,8 @@ def _transform_run(course_run: dict, course: dict) -> dict: ], "availability": course["availability"], "delivery": transform_delivery(course.get("format")), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } @@ -158,6 +162,8 @@ def _transform_learning_resource_course(course): "certification_type": CertificationType.professional.name, "availability": course["availability"], "continuing_ed_credits": course["credits"], + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } @@ -216,6 +222,8 @@ def transform_programs(programs): ], "delivery": transform_delivery(program.get("format")), "availability": program["availability"], + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } ], "courses": transform_courses(program["courses"]), @@ -223,6 +231,8 @@ def transform_programs(programs): "certification_type": CertificationType.professional.name, "availability": program["availability"], "continuing_ed_credits": program["credits"], + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for program in programs ] diff --git a/learning_resources/etl/xpro_test.py b/learning_resources/etl/xpro_test.py index 0f1954a5c2..7acb9e8226 100644 --- a/learning_resources/etl/xpro_test.py +++ b/learning_resources/etl/xpro_test.py @@ -10,7 +10,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, + Pace, PlatformType, ) from learning_resources.etl import xpro @@ -112,6 +114,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): "resource_type": LearningResourceType.program.name, "delivery": transform_delivery(program_data.get("format")), "continuing_ed_credits": program_data.get("credits"), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], "runs": [ { "run_id": program_data["readable_id"], @@ -131,6 +135,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): "description": program_data["description"], "delivery": transform_delivery(program_data.get("format")), "availability": Availability.dated.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } ], "courses": [ @@ -153,6 +159,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): "topics": parse_topics(course_data), "resource_type": LearningResourceType.course.name, "continuing_ed_credits": course_data.get("credits"), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], "runs": [ { "run_id": course_run_data["courseware_id"], @@ -173,6 +181,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): ], "delivery": transform_delivery(course_data.get("format")), "availability": Availability.dated.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for course_run_data in course_data["courseruns"] ], @@ -245,6 +255,8 @@ def test_xpro_transform_courses(mock_xpro_courses_data): ], "delivery": transform_delivery(course_data.get("format")), "availability": Availability.dated.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for course_run_data in course_data["courseruns"] ], @@ -262,6 +274,8 @@ def test_xpro_transform_courses(mock_xpro_courses_data): "certification": True, "certification_type": CertificationType.professional.name, "continuing_ed_credits": course_data.get("credits"), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for course_data in mock_xpro_courses_data ] diff --git a/learning_resources/migrations/0068_learningresource_format_pace.py b/learning_resources/migrations/0068_learningresource_format_pace.py new file mode 100644 index 0000000000..6929be78d1 --- /dev/null +++ b/learning_resources/migrations/0068_learningresource_format_pace.py @@ -0,0 +1,75 @@ +# Generated by Django 4.2.16 on 2024-09-23 14:18 + +import django.contrib.postgres.fields +from django.db import migrations, models + +import learning_resources.models + + +class Migration(migrations.Migration): + dependencies = [ + ("learning_resources", "0067_ocw_delivery_online_only"), + ] + + operations = [ + migrations.AddField( + model_name="learningresource", + name="format", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("synchronous", "Synchronous"), + ("asynchronous", "Asynchronous"), + ], + max_length=24, + ), + default=learning_resources.models.default_format, + size=None, + ), + ), + migrations.AddField( + model_name="learningresource", + name="pace", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("self_paced", "Self-paced"), + ("instructor_paced", "Instructor-paced"), + ], + max_length=24, + ), + default=learning_resources.models.default_pace, + size=None, + ), + ), + migrations.AddField( + model_name="learningresourcerun", + name="format", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("synchronous", "Synchronous"), + ("asynchronous", "Asynchronous"), + ], + max_length=24, + ), + default=learning_resources.models.default_format, + size=None, + ), + ), + migrations.AddField( + model_name="learningresourcerun", + name="pace", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("self_paced", "Self-paced"), + ("instructor_paced", "Instructor-paced"), + ], + max_length=24, + ), + default=learning_resources.models.default_pace, + size=None, + ), + ), + ] diff --git a/learning_resources/models.py b/learning_resources/models.py index 069214b7a6..4b74e723cd 100644 --- a/learning_resources/models.py +++ b/learning_resources/models.py @@ -16,9 +16,11 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceDelivery, LearningResourceRelationTypes, LearningResourceType, + Pace, PrivacyLevel, ) from main.models import TimestampedModel, TimestampedModelQuerySet @@ -29,6 +31,16 @@ def default_delivery(): return [LearningResourceDelivery.online.name] +def default_pace(): + """Return the default pace as a list""" + return [Pace.self_paced.name] + + +def default_format(): + """Return the default format as a list""" + return [Format.asynchronous.name] + + class LearningResourcePlatform(TimestampedModel): """Platforms for all learning resources""" @@ -428,6 +440,20 @@ class LearningResource(TimestampedModel): continuing_ed_credits = models.DecimalField( max_digits=5, decimal_places=2, null=True, blank=True ) + pace = ArrayField( + models.CharField( + max_length=24, + choices=Pace.as_tuple(), + ), + default=default_pace, + ) + format = ArrayField( + models.CharField( + max_length=24, + choices=Format.as_tuple(), + ), + default=default_format, + ) @property def audience(self) -> str | None: @@ -565,6 +591,20 @@ class LearningResourceRun(TimestampedModel): null=True, choices=Availability.as_tuple(), ) + pace = ArrayField( + models.CharField( + max_length=24, + choices=Pace.as_tuple(), + ), + default=default_pace, + ) + format = ArrayField( + models.CharField( + max_length=24, + choices=Format.as_tuple(), + ), + default=default_format, + ) def __str__(self): return f"LearningResourceRun platform={self.learning_resource.platform} run_id={self.run_id}" # noqa: E501 diff --git a/learning_resources/serializers.py b/learning_resources/serializers.py index 64b76380eb..c52a40ebc6 100644 --- a/learning_resources/serializers.py +++ b/learning_resources/serializers.py @@ -15,9 +15,11 @@ from learning_resources.constants import ( LEARNING_MATERIAL_RESOURCE_CATEGORY, CertificationType, + Format, LearningResourceDelivery, LearningResourceType, LevelType, + Pace, ) from learning_resources.etl.loaders import update_index from main.serializers import COMMON_IGNORED_FIELDS, WriteableSerializerMethodField @@ -236,6 +238,36 @@ def to_representation(self, value): return {"code": value, "name": LearningResourceDelivery[value].value} +@extend_schema_field( + { + "type": "object", + "properties": { + "code": {"enum": Format.names()}, + "name": {"type": "string"}, + }, + "required": ["code", "name"], + } +) +class FormatSerializer(serializers.Field): + def to_representation(self, value): + return {"code": value, "name": Format[value].value} + + +@extend_schema_field( + { + "type": "object", + "properties": { + "code": {"enum": Pace.names()}, + "name": {"type": "string"}, + }, + "required": ["code", "name"], + } +) +class PaceSerializer(serializers.Field): + def to_representation(self, value): + return {"code": value, "name": Pace[value].value} + + class LearningResourceRunSerializer(serializers.ModelSerializer): """Serializer for the LearningResourceRun model""" @@ -248,6 +280,8 @@ class LearningResourceRunSerializer(serializers.ModelSerializer): delivery = serializers.ListField( child=LearningResourceDeliverySerializer(), read_only=True ) + format = serializers.ListField(child=FormatSerializer(), read_only=True) + pace = serializers.ListField(child=PaceSerializer(), read_only=True) class Meta: model = models.LearningResourceRun @@ -415,6 +449,8 @@ class LearningResourceBaseSerializer(serializers.ModelSerializer, WriteableTopic ) free = serializers.SerializerMethodField() resource_category = serializers.SerializerMethodField() + format = serializers.ListField(child=FormatSerializer(), read_only=True) + pace = serializers.ListField(child=PaceSerializer(), read_only=True) def get_resource_category(self, instance) -> str: """Return the resource category of the resource""" diff --git a/learning_resources/serializers_test.py b/learning_resources/serializers_test.py index 3ca30f06dc..bc68711f96 100644 --- a/learning_resources/serializers_test.py +++ b/learning_resources/serializers_test.py @@ -12,9 +12,11 @@ from learning_resources.constants import ( LEARNING_MATERIAL_RESOURCE_CATEGORY, CertificationType, + Format, LearningResourceDelivery, LearningResourceRelationTypes, LearningResourceType, + Pace, PlatformType, ) from learning_resources.models import ContentFile, LearningResource @@ -262,6 +264,13 @@ def test_learning_resource_serializer( # noqa: PLR0913 {"code": lr_delivery, "name": LearningResourceDelivery[lr_delivery].value} for lr_delivery in resource.delivery ], + "format": [ + {"code": lr_format, "name": Format[lr_format].value} + for lr_format in resource.format + ], + "pace": [ + {"code": lr_pace, "name": Pace[lr_pace].value} for lr_pace in resource.pace + ], "next_start_date": resource.next_start_date, "availability": resource.availability, "completeness": 1.0, diff --git a/learning_resources_search/constants.py b/learning_resources_search/constants.py index b0b1d53051..a4208aaf4e 100644 --- a/learning_resources_search/constants.py +++ b/learning_resources_search/constants.py @@ -121,6 +121,20 @@ class FilterConfig: "name": {"type": "keyword"}, }, }, + "pace": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, + "format": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, "readable_id": {"type": "keyword"}, "title": ENGLISH_TEXT_FIELD_WITH_SUGGEST, "description": ENGLISH_TEXT_FIELD_WITH_SUGGEST, @@ -231,6 +245,20 @@ class FilterConfig: "name": {"type": "keyword"}, }, }, + "pace": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, + "format": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, "semester": {"type": "keyword"}, "year": {"type": "keyword"}, "start_date": {"type": "date"}, diff --git a/main/models.py b/main/models.py index 686cea755a..4bafc925ff 100644 --- a/main/models.py +++ b/main/models.py @@ -18,7 +18,7 @@ def update(self, **kwargs): Automatically update updated_on timestamp when .update(). This is because .update() does not go through .save(), thus will not auto_now, because it happens on the database level without loading objects into memory. - """ # noqa: D402, E501 + """ # noqa: E501 if "updated_on" not in kwargs: kwargs["updated_on"] = now_in_utc() return super().update(**kwargs) diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index ea686f47a4..2f6057ec71 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -7956,6 +7956,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/CourseResourceResourceTypeEnum' @@ -8023,11 +8053,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional @@ -8398,6 +8430,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/LearningPathResourceResourceTypeEnum' @@ -8463,12 +8525,14 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path - learning_path_parents - offered_by + - pace - platform - prices - readable_id @@ -8964,6 +9028,36 @@ components: - code - name readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true run_id: type: string maxLength: 128 @@ -9040,10 +9134,12 @@ components: - $ref: '#/components/schemas/NullEnum' required: - delivery + - format - id - image - instructors - level + - pace - run_id - title LearningResourceRunRequest: @@ -10405,6 +10501,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/PodcastEpisodeResourceResourceTypeEnum' @@ -10471,11 +10597,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - podcast_episode - prices @@ -10687,6 +10815,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/PodcastResourceResourceTypeEnum' @@ -10753,11 +10911,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - podcast - prices @@ -11099,6 +11259,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/ProgramResourceResourceTypeEnum' @@ -11165,11 +11355,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional @@ -11653,6 +11845,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/VideoPlaylistResourceResourceTypeEnum' @@ -11719,11 +11941,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional @@ -11921,6 +12145,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/VideoResourceResourceTypeEnum' @@ -11987,11 +12241,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional From 58b85032732b3bd4574b5a62e8b5cc98ff74f040 Mon Sep 17 00:00:00 2001 From: Anastasia Beglova Date: Tue, 24 Sep 2024 11:57:56 -0400 Subject: [PATCH 10/13] new -> recently added (#1594) --- .../mit-learn/src/page-components/Header/Header.tsx | 2 +- .../page-components/SearchDisplay/SearchDisplay.tsx | 10 +++------- .../pages/ChannelPage/ChannelSearchFacetDisplay.tsx | 5 ++--- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/frontends/mit-learn/src/page-components/Header/Header.tsx b/frontends/mit-learn/src/page-components/Header/Header.tsx index 1f7cf88f89..a0756c3d86 100644 --- a/frontends/mit-learn/src/page-components/Header/Header.tsx +++ b/frontends/mit-learn/src/page-components/Header/Header.tsx @@ -219,7 +219,7 @@ const navData: NavData = { title: "DISCOVER LEARNING RESOURCES", items: [ { - title: "New", + title: "Recently Added", icon: , href: SEARCH_NEW, }, diff --git a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx index 7c27760057..704261b0b8 100644 --- a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -52,10 +52,6 @@ import { ResourceCard } from "../ResourceCard/ResourceCard" import { useSearchParams } from "@mitodl/course-search-utils/react-router" import { useUserMe } from "api/hooks/user" -export const StyledSelect = styled(SimpleSelect)` - min-width: 160px; -` - const StyledResourceTabs = styled(ResourceCategoryTabs.TabList)` margin-top: 0 px; ` @@ -470,7 +466,7 @@ const SORT_OPTIONS = [ value: "", }, { - label: "New", + label: "Recently Added", value: "new", }, { @@ -596,7 +592,7 @@ const SearchDisplay: React.FC = ({ } const searchModeDropdown = ( - @@ -615,7 +611,7 @@ const SearchDisplay: React.FC = ({ ) const sortDropdown = ( - setParamValue("sortby", e.target.value)} diff --git a/frontends/mit-learn/src/pages/ChannelPage/ChannelSearchFacetDisplay.tsx b/frontends/mit-learn/src/pages/ChannelPage/ChannelSearchFacetDisplay.tsx index 2da87b65d5..c5404b191e 100644 --- a/frontends/mit-learn/src/pages/ChannelPage/ChannelSearchFacetDisplay.tsx +++ b/frontends/mit-learn/src/pages/ChannelPage/ChannelSearchFacetDisplay.tsx @@ -8,9 +8,8 @@ import type { BooleanFacetKey, } from "@mitodl/course-search-utils" import { BOOLEAN_FACET_NAMES } from "@mitodl/course-search-utils" -import { Skeleton, styled } from "ol-components" +import { Skeleton, styled, SimpleSelect } from "ol-components" import type { SimpleSelectOption } from "ol-components" -import { StyledSelect } from "@/page-components/SearchDisplay/SearchDisplay" const StyledSkeleton = styled(Skeleton)` display: inline-flex; @@ -117,7 +116,7 @@ const AvailableFacetsDropdowns: React.FC< return ( facetItems.length && ( - Date: Tue, 24 Sep 2024 16:03:37 +0000 Subject: [PATCH 11/13] Release 0.19.4 --- RELEASE.rst | 6 ++++++ main/settings.py | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index 1ae708ba43..c9493d04b6 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,12 @@ Release Notes ============= +Version 0.19.4 +-------------- + +- new -> recently added (#1594) +- Pace and format fields for learning resources (#1588) + Version 0.19.3 (Released September 23, 2024) -------------- diff --git a/main/settings.py b/main/settings.py index c4e3864445..8941615e60 100644 --- a/main/settings.py +++ b/main/settings.py @@ -33,7 +33,7 @@ from main.settings_pluggy import * # noqa: F403 from openapi.settings_spectacular import open_spectacular_settings -VERSION = "0.19.3" +VERSION = "0.19.4" log = logging.getLogger() From 28dccfec56b4c68881e2796a9f46bd507e86195b Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Tue, 24 Sep 2024 17:13:57 -0400 Subject: [PATCH 12/13] Add separate field for ocw topics, use best field to assign related topics (#1600) --- frontends/api/src/generated/v1/api.ts | 186 ++++++++++++++++++ learning_resources/etl/ocw.py | 47 ++++- learning_resources/etl/ocw_test.py | 56 +++++- .../0069_learningresource_ocw_topics.py | 23 +++ learning_resources/models.py | 1 + learning_resources/serializers_test.py | 1 + learning_resources_search/constants.py | 2 + learning_resources_search/serializers.py | 5 + openapi/specs/v1.yaml | 129 ++++++++++++ .../data.json | 12 +- 10 files changed, 450 insertions(+), 12 deletions(-) create mode 100644 learning_resources/migrations/0069_learningresource_ocw_topics.py diff --git a/frontends/api/src/generated/v1/api.ts b/frontends/api/src/generated/v1/api.ts index ff24f6357f..cf9b817ed5 100644 --- a/frontends/api/src/generated/v1/api.ts +++ b/frontends/api/src/generated/v1/api.ts @@ -758,6 +758,12 @@ export interface CourseResource { * @memberof CourseResource */ url?: string | null + /** + * + * @type {Array} + * @memberof CourseResource + */ + ocw_topics?: Array /** * * @type {boolean} @@ -972,6 +978,12 @@ export interface CourseResourceRequest { * @memberof CourseResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof CourseResourceRequest + */ + ocw_topics?: Array /** * * @type {string} @@ -1516,6 +1528,12 @@ export interface LearningPathResource { * @memberof LearningPathResource */ url?: string | null + /** + * + * @type {Array} + * @memberof LearningPathResource + */ + ocw_topics?: Array /** * * @type {boolean} @@ -1608,6 +1626,12 @@ export interface LearningPathResourceRequest { * @memberof LearningPathResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof LearningPathResourceRequest + */ + ocw_topics?: Array /** * * @type {boolean} @@ -3439,6 +3463,12 @@ export interface PatchedLearningPathResourceRequest { * @memberof PatchedLearningPathResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof PatchedLearningPathResourceRequest + */ + ocw_topics?: Array /** * * @type {boolean} @@ -3652,6 +3682,12 @@ export interface PercolateQuerySubscriptionRequestRequest { * @memberof PercolateQuerySubscriptionRequestRequest */ topic?: Array + /** + * The ocw topic name. + * @type {Array} + * @memberof PercolateQuerySubscriptionRequestRequest + */ + ocw_topic?: Array /** * If true return raw open search results with score explanations * @type {boolean} @@ -4213,6 +4249,12 @@ export interface PodcastEpisodeResource { * @memberof PodcastEpisodeResource */ url?: string | null + /** + * + * @type {Array} + * @memberof PodcastEpisodeResource + */ + ocw_topics?: Array /** * * @type {boolean} @@ -4311,6 +4353,12 @@ export interface PodcastEpisodeResourceRequest { * @memberof PodcastEpisodeResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof PodcastEpisodeResourceRequest + */ + ocw_topics?: Array /** * * @type {string} @@ -4571,6 +4619,12 @@ export interface PodcastResource { * @memberof PodcastResource */ url?: string | null + /** + * + * @type {Array} + * @memberof PodcastResource + */ + ocw_topics?: Array /** * * @type {boolean} @@ -4669,6 +4723,12 @@ export interface PodcastResourceRequest { * @memberof PodcastResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof PodcastResourceRequest + */ + ocw_topics?: Array /** * * @type {string} @@ -5149,6 +5209,12 @@ export interface ProgramResource { * @memberof ProgramResource */ url?: string | null + /** + * + * @type {Array} + * @memberof ProgramResource + */ + ocw_topics?: Array /** * * @type {boolean} @@ -5247,6 +5313,12 @@ export interface ProgramResourceRequest { * @memberof ProgramResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof ProgramResourceRequest + */ + ocw_topics?: Array /** * * @type {string} @@ -5986,6 +6058,12 @@ export interface VideoPlaylistResource { * @memberof VideoPlaylistResource */ url?: string | null + /** + * + * @type {Array} + * @memberof VideoPlaylistResource + */ + ocw_topics?: Array /** * * @type {boolean} @@ -6084,6 +6162,12 @@ export interface VideoPlaylistResourceRequest { * @memberof VideoPlaylistResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof VideoPlaylistResourceRequest + */ + ocw_topics?: Array /** * * @type {string} @@ -6332,6 +6416,12 @@ export interface VideoResource { * @memberof VideoResource */ url?: string | null + /** + * + * @type {Array} + * @memberof VideoResource + */ + ocw_topics?: Array /** * * @type {boolean} @@ -6430,6 +6520,12 @@ export interface VideoResourceRequest { * @memberof VideoResourceRequest */ url?: string | null + /** + * + * @type {Array} + * @memberof VideoResourceRequest + */ + ocw_topics?: Array /** * * @type {string} @@ -7190,6 +7286,7 @@ export const ContentFileSearchApiAxiosParamCreator = function ( * @param {boolean | null} [dev_mode] If true return raw open search results with score explanations * @param {Array} [id] The id value for the content file * @param {number} [limit] Number of results to return per page + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -7207,6 +7304,7 @@ export const ContentFileSearchApiAxiosParamCreator = function ( dev_mode?: boolean | null, id?: Array, limit?: number, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -7253,6 +7351,10 @@ export const ContentFileSearchApiAxiosParamCreator = function ( localVarQueryParameter["limit"] = limit } + if (ocw_topic) { + localVarQueryParameter["ocw_topic"] = ocw_topic + } + if (offered_by) { localVarQueryParameter["offered_by"] = offered_by } @@ -7318,6 +7420,7 @@ export const ContentFileSearchApiFp = function (configuration?: Configuration) { * @param {boolean | null} [dev_mode] If true return raw open search results with score explanations * @param {Array} [id] The id value for the content file * @param {number} [limit] Number of results to return per page + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -7335,6 +7438,7 @@ export const ContentFileSearchApiFp = function (configuration?: Configuration) { dev_mode?: boolean | null, id?: Array, limit?: number, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -7357,6 +7461,7 @@ export const ContentFileSearchApiFp = function (configuration?: Configuration) { dev_mode, id, limit, + ocw_topic, offered_by, offset, platform, @@ -7412,6 +7517,7 @@ export const ContentFileSearchApiFactory = function ( requestParameters.dev_mode, requestParameters.id, requestParameters.limit, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -7468,6 +7574,13 @@ export interface ContentFileSearchApiContentFileSearchRetrieveRequest { */ readonly limit?: number + /** + * The ocw topic name. + * @type {Array} + * @memberof ContentFileSearchApiContentFileSearchRetrieve + */ + readonly ocw_topic?: Array + /** * The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @type {Array<'mitx' | 'ocw' | 'bootcamps' | 'xpro' | 'mitpe' | 'see'>} @@ -7551,6 +7664,7 @@ export class ContentFileSearchApi extends BaseAPI { requestParameters.dev_mode, requestParameters.id, requestParameters.limit, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -12619,6 +12733,7 @@ export const LearningResourcesSearchApiAxiosParamCreator = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -12648,6 +12763,7 @@ export const LearningResourcesSearchApiAxiosParamCreator = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -12731,6 +12847,10 @@ export const LearningResourcesSearchApiAxiosParamCreator = function ( localVarQueryParameter["min_score"] = min_score } + if (ocw_topic) { + localVarQueryParameter["ocw_topic"] = ocw_topic + } + if (offered_by) { localVarQueryParameter["offered_by"] = offered_by } @@ -12822,6 +12942,7 @@ export const LearningResourcesSearchApiFp = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -12851,6 +12972,7 @@ export const LearningResourcesSearchApiFp = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -12885,6 +13007,7 @@ export const LearningResourcesSearchApiFp = function ( limit, max_incompleteness_penalty, min_score, + ocw_topic, offered_by, offset, platform, @@ -12952,6 +13075,7 @@ export const LearningResourcesSearchApiFactory = function ( requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -13068,6 +13192,13 @@ export interface LearningResourcesSearchApiLearningResourcesSearchRetrieveReques */ readonly min_score?: number | null + /** + * The ocw topic name. + * @type {Array} + * @memberof LearningResourcesSearchApiLearningResourcesSearchRetrieve + */ + readonly ocw_topic?: Array + /** * The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @type {Array<'mitx' | 'ocw' | 'bootcamps' | 'xpro' | 'mitpe' | 'see'>} @@ -13187,6 +13318,7 @@ export class LearningResourcesSearchApi extends BaseAPI { requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -13424,6 +13556,7 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -13454,6 +13587,7 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -13538,6 +13672,10 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( localVarQueryParameter["min_score"] = min_score } + if (ocw_topic) { + localVarQueryParameter["ocw_topic"] = ocw_topic + } + if (offered_by) { localVarQueryParameter["offered_by"] = offered_by } @@ -13620,6 +13758,7 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -13649,6 +13788,7 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -13732,6 +13872,10 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( localVarQueryParameter["min_score"] = min_score } + if (ocw_topic) { + localVarQueryParameter["ocw_topic"] = ocw_topic + } + if (offered_by) { localVarQueryParameter["offered_by"] = offered_by } @@ -13810,6 +13954,7 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -13841,6 +13986,7 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -13926,6 +14072,10 @@ export const LearningResourcesUserSubscriptionApiAxiosParamCreator = function ( localVarQueryParameter["min_score"] = min_score } + if (ocw_topic) { + localVarQueryParameter["ocw_topic"] = ocw_topic + } + if (offered_by) { localVarQueryParameter["offered_by"] = offered_by } @@ -14079,6 +14229,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -14109,6 +14260,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -14144,6 +14296,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( limit, max_incompleteness_penalty, min_score, + ocw_topic, offered_by, offset, platform, @@ -14188,6 +14341,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -14217,6 +14371,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -14251,6 +14406,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( limit, max_incompleteness_penalty, min_score, + ocw_topic, offered_by, offset, platform, @@ -14294,6 +14450,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( * @param {number} [limit] Number of results to return per page * @param {number | null} [max_incompleteness_penalty] Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. Only affects results if there is a search term. * @param {number | null} [min_score] Minimum score value a text query result needs to have to be displayed + * @param {Array} [ocw_topic] The ocw topic name. * @param {Array} [offered_by] The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @param {number} [offset] The initial index from which to return the results * @param {Array} [platform] The platform on which the learning resource is offered * `edx` - edX * `ocw` - MIT OpenCourseWare * `oll` - Open Learning Library * `mitxonline` - MITx Online * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `csail` - CSAIL * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * `scc` - Schwarzman College of Computing * `ctl` - Center for Transportation & Logistics * `whu` - WHU * `susskind` - Susskind * `globalalumni` - Global Alumni * `simplilearn` - Simplilearn * `emeritus` - Emeritus * `podcast` - Podcast * `youtube` - YouTube @@ -14325,6 +14482,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( limit?: number, max_incompleteness_penalty?: number | null, min_score?: number | null, + ocw_topic?: Array, offered_by?: Array, offset?: number, platform?: Array, @@ -14358,6 +14516,7 @@ export const LearningResourcesUserSubscriptionApiFp = function ( limit, max_incompleteness_penalty, min_score, + ocw_topic, offered_by, offset, platform, @@ -14458,6 +14617,7 @@ export const LearningResourcesUserSubscriptionApiFactory = function ( requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -14501,6 +14661,7 @@ export const LearningResourcesUserSubscriptionApiFactory = function ( requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -14543,6 +14704,7 @@ export const LearningResourcesUserSubscriptionApiFactory = function ( requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -14679,6 +14841,13 @@ export interface LearningResourcesUserSubscriptionApiLearningResourcesUserSubscr */ readonly min_score?: number | null + /** + * The ocw topic name. + * @type {Array} + * @memberof LearningResourcesUserSubscriptionApiLearningResourcesUserSubscriptionCheckList + */ + readonly ocw_topic?: Array + /** * The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @type {Array<'mitx' | 'ocw' | 'bootcamps' | 'xpro' | 'mitpe' | 'see'>} @@ -14868,6 +15037,13 @@ export interface LearningResourcesUserSubscriptionApiLearningResourcesUserSubscr */ readonly min_score?: number | null + /** + * The ocw topic name. + * @type {Array} + * @memberof LearningResourcesUserSubscriptionApiLearningResourcesUserSubscriptionList + */ + readonly ocw_topic?: Array + /** * The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @type {Array<'mitx' | 'ocw' | 'bootcamps' | 'xpro' | 'mitpe' | 'see'>} @@ -15050,6 +15226,13 @@ export interface LearningResourcesUserSubscriptionApiLearningResourcesUserSubscr */ readonly min_score?: number | null + /** + * The ocw topic name. + * @type {Array} + * @memberof LearningResourcesUserSubscriptionApiLearningResourcesUserSubscriptionSubscribeCreate + */ + readonly ocw_topic?: Array + /** * The organization that offers the learning resource * `mitx` - MITx * `ocw` - MIT OpenCourseWare * `bootcamps` - Bootcamps * `xpro` - MIT xPRO * `mitpe` - MIT Professional Education * `see` - MIT Sloan Executive Education * @type {Array<'mitx' | 'ocw' | 'bootcamps' | 'xpro' | 'mitpe' | 'see'>} @@ -15197,6 +15380,7 @@ export class LearningResourcesUserSubscriptionApi extends BaseAPI { requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -15242,6 +15426,7 @@ export class LearningResourcesUserSubscriptionApi extends BaseAPI { requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, @@ -15286,6 +15471,7 @@ export class LearningResourcesUserSubscriptionApi extends BaseAPI { requestParameters.limit, requestParameters.max_incompleteness_penalty, requestParameters.min_score, + requestParameters.ocw_topic, requestParameters.offered_by, requestParameters.offset, requestParameters.platform, diff --git a/learning_resources/etl/ocw.py b/learning_resources/etl/ocw.py index d776f63c82..31cac2e2fb 100644 --- a/learning_resources/etl/ocw.py +++ b/learning_resources/etl/ocw.py @@ -71,6 +71,40 @@ def parse_delivery(course_data: dict) -> list[str]: return delivery +def parse_learn_topics(course_data: dict) -> list[dict]: + """ + Parse topics. Use the "mit_learn_topics" field if it exists and isn't empty, + otherwise use and transform the "topics" field values. + + Args: + course_data (dict): The course data + + Returns: + list[dict]: The topics + """ + mitlearn_topics = course_data.get("mit_learn_topics") or [] + ocw_topics = course_data.get("topics") or [] + if mitlearn_topics: + # Should already be in the correct format + return [ + {"name": topic_name} + for topic_name in sorted( + {topic for topics in mitlearn_topics for topic in topics} + ) + ] + else: + # Topics need to be transformed + return transform_topics( + [ + {"name": topic_name} + for topic_name in sorted( + {topic for topics in ocw_topics for topic in topics} + ) + ], + OFFERED_BY["code"], + ) + + def transform_content_files( s3_resource: boto3.resource, course_prefix: str, @@ -329,10 +363,6 @@ def transform_course(course_data: dict) -> dict: readable_term = f"+{slugify(term)}" if term else "" readable_year = f"_{course_data.get('year')}" if year else "" readable_id = f"{course_data[PRIMARY_COURSE_ID]}{readable_term}{readable_year}" - topics = transform_topics( - [{"name": topic} for topics in course_data.get("topics") for topic in topics], - OFFERED_BY["code"], - ) image_src = course_data.get("image_src") return { @@ -365,7 +395,14 @@ def transform_course(course_data: dict) -> dict: is_ocw=True, ), }, - "topics": topics, + "topics": parse_learn_topics(course_data), + "ocw_topics": sorted( + { + topic_name + for topic_sublist in course_data.get("topics", []) + for topic_name in topic_sublist + } + ), "runs": [transform_run(course_data)], "resource_type": LearningResourceType.course.name, "unique_field": UNIQUE_FIELD, diff --git a/learning_resources/etl/ocw_test.py b/learning_resources/etl/ocw_test.py index d54b562ce7..7ffba2cf4c 100644 --- a/learning_resources/etl/ocw_test.py +++ b/learning_resources/etl/ocw_test.py @@ -18,11 +18,15 @@ ) from learning_resources.etl.constants import CourseNumberType, ETLSource from learning_resources.etl.ocw import ( + parse_learn_topics, transform_content_files, transform_contentfile, transform_course, ) -from learning_resources.factories import ContentFileFactory +from learning_resources.factories import ( + ContentFileFactory, + LearningResourceTopicFactory, +) from learning_resources.models import ContentFile from learning_resources.utils import ( get_s3_object_and_read, @@ -239,6 +243,14 @@ def test_transform_course( # noqa: PLR0913 ) transformed_json = transform_course(extracted_json) if expected_uid: + assert transformed_json["ocw_topics"] == [ + "Anthropology", + "Ethnography", + "Humanities", + "Philosophy", + "Political Philosophy", + "Social Science", + ] assert transformed_json["readable_id"] == expected_id assert transformed_json["etl_source"] == ETLSource.ocw.name assert transformed_json["delivery"] == expected_delivery @@ -295,3 +307,45 @@ def test_transform_course( # noqa: PLR0913 ) else: assert transformed_json is None + + +@pytest.mark.parametrize("has_learn_topics", [True, False]) +def test_parse_topics(mocker, has_learn_topics): + """Topics should be assigned correctly based on mitlearn topics if present, ocw topics if not""" + ocw_topics = [ + ["Social Science", "Anthropology", "Ethnography"], + ["Social Science", "Political Science", "International Relations"], + ] + mit_learn_topics = ( + [["Social Sciences", "Anthropology"], ["Social Sciences", "Political Science"]] + if has_learn_topics + else [] + ) + course_data = { + "topics": ocw_topics, + "mit_learn_topics": mit_learn_topics, + } + mocker.patch( + "learning_resources.etl.utils.load_offeror_topic_map", + return_value={ + "Political Philosophy": ["Philosophy"], + "Ethnography": ["Anthropology"], + "International Relations": ["Political Science"], + }, + ) + for topic in ("Social Sciences", "Anthropology", "Political Science"): + LearningResourceTopicFactory.create(name=topic) + topics_dict = parse_learn_topics(course_data) + if has_learn_topics: + assert topics_dict == [ + {"name": "Anthropology"}, + {"name": "Political Science"}, + {"name": "Social Sciences"}, + ] + else: + assert topics_dict == [ + {"name": "Anthropology"}, + {"name": "Anthropology"}, + {"name": "Political Science"}, + {"name": "Political Science"}, + ] diff --git a/learning_resources/migrations/0069_learningresource_ocw_topics.py b/learning_resources/migrations/0069_learningresource_ocw_topics.py new file mode 100644 index 0000000000..3ee3678b5a --- /dev/null +++ b/learning_resources/migrations/0069_learningresource_ocw_topics.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.16 on 2024-09-23 18:04 + +import django.contrib.postgres.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("learning_resources", "0068_learningresource_format_pace"), + ] + + operations = [ + migrations.AddField( + model_name="learningresource", + name="ocw_topics", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField(max_length=128), + blank=True, + default=list, + size=None, + ), + ), + ] diff --git a/learning_resources/models.py b/learning_resources/models.py index 4b74e723cd..e3c3bf3d75 100644 --- a/learning_resources/models.py +++ b/learning_resources/models.py @@ -411,6 +411,7 @@ class LearningResource(TimestampedModel): choices=((member.name, member.value) for member in LearningResourceType), ) topics = models.ManyToManyField(LearningResourceTopic) + ocw_topics = ArrayField(models.CharField(max_length=128), default=list, blank=True) offered_by = models.ForeignKey( LearningResourceOfferor, null=True, on_delete=models.SET_NULL ) diff --git a/learning_resources/serializers_test.py b/learning_resources/serializers_test.py index bc68711f96..b454de5ee7 100644 --- a/learning_resources/serializers_test.py +++ b/learning_resources/serializers_test.py @@ -254,6 +254,7 @@ def test_learning_resource_serializer( # noqa: PLR0913 serializers.LearningResourceTopicSerializer(topic).data for topic in resource.topics.all() ], + "ocw_topics": sorted(resource.ocw_topics), "runs": [ serializers.LearningResourceRunSerializer(instance=run).data for run in resource.runs.all() diff --git a/learning_resources_search/constants.py b/learning_resources_search/constants.py index a4208aaf4e..6bcc47e1f0 100644 --- a/learning_resources_search/constants.py +++ b/learning_resources_search/constants.py @@ -71,6 +71,7 @@ class FilterConfig: "run_id": FilterConfig("run_id", case_sensitive=True), "resource_id": FilterConfig("resource_id", case_sensitive=True), "topic": FilterConfig("topics.name"), + "ocw_topic": FilterConfig("ocw_topics"), "level": FilterConfig("runs.level.code"), "department": FilterConfig("departments.department_id"), "platform": FilterConfig("platform.code"), @@ -184,6 +185,7 @@ class FilterConfig: "channel_url": {"type": "keyword"}, }, }, + "ocw_topics": {"type": "keyword"}, "offered_by": { "type": "nested", "properties": { diff --git a/learning_resources_search/serializers.py b/learning_resources_search/serializers.py index fd9bd3fdcf..b340605169 100644 --- a/learning_resources_search/serializers.py +++ b/learning_resources_search/serializers.py @@ -276,6 +276,11 @@ class SearchRequestSerializer(serializers.Serializer): child=serializers.CharField(), help_text="The topic name. To see a list of options go to api/v1/topics/", ) + ocw_topic = serializers.ListField( + required=False, + child=serializers.CharField(), + help_text="The ocw topic name.", + ) dev_mode = serializers.BooleanField( required=False, allow_null=True, diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index 2f6057ec71..af71655a91 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -175,6 +175,14 @@ paths: schema: type: integer description: Number of results to return per page + - in: query + name: ocw_topic + schema: + type: array + items: + type: string + minLength: 1 + description: The ocw topic name. - in: query name: offered_by schema: @@ -2506,6 +2514,14 @@ paths: minimum: 0 nullable: true description: Minimum score value a text query result needs to have to be displayed + - in: query + name: ocw_topic + schema: + type: array + items: + type: string + minLength: 1 + description: The ocw topic name. - in: query name: offered_by schema: @@ -2986,6 +3002,14 @@ paths: minimum: 0 nullable: true description: Minimum score value a text query result needs to have to be displayed + - in: query + name: ocw_topic + schema: + type: array + items: + type: string + minLength: 1 + description: The ocw topic name. - in: query name: offered_by schema: @@ -3491,6 +3515,14 @@ paths: minimum: 0 nullable: true description: Minimum score value a text query result needs to have to be displayed + - in: query + name: ocw_topic + schema: + type: array + items: + type: string + minLength: 1 + description: The ocw topic name. - in: query name: offered_by schema: @@ -3987,6 +4019,14 @@ paths: minimum: 0 nullable: true description: Minimum score value a text query result needs to have to be displayed + - in: query + name: ocw_topic + schema: + type: array + items: + type: string + minLength: 1 + description: The ocw topic name. - in: query name: offered_by schema: @@ -8024,6 +8064,11 @@ components: format: uri nullable: true maxLength: 2048 + ocw_topics: + type: array + items: + type: string + maxLength: 128 professional: type: boolean readOnly: true @@ -8111,6 +8156,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 next_start_date: type: string format: date-time @@ -8498,6 +8549,11 @@ components: format: uri nullable: true maxLength: 2048 + ocw_topics: + type: array + items: + type: string + maxLength: 128 professional: type: boolean next_start_date: @@ -8579,6 +8635,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 professional: type: boolean next_start_date: @@ -9958,6 +10020,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 professional: type: boolean next_start_date: @@ -10088,6 +10156,12 @@ components: type: string minLength: 1 description: The topic name. To see a list of options go to api/v1/topics/ + ocw_topic: + type: array + items: + type: string + minLength: 1 + description: The ocw topic name. dev_mode: type: boolean nullable: true @@ -10569,6 +10643,11 @@ components: format: uri nullable: true maxLength: 2048 + ocw_topics: + type: array + items: + type: string + maxLength: 128 professional: type: boolean readOnly: true @@ -10656,6 +10735,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 next_start_date: type: string format: date-time @@ -10883,6 +10968,11 @@ components: format: uri nullable: true maxLength: 2048 + ocw_topics: + type: array + items: + type: string + maxLength: 128 professional: type: boolean readOnly: true @@ -10970,6 +11060,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 next_start_date: type: string format: date-time @@ -11327,6 +11423,11 @@ components: format: uri nullable: true maxLength: 2048 + ocw_topics: + type: array + items: + type: string + maxLength: 128 professional: type: boolean readOnly: true @@ -11414,6 +11515,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 next_start_date: type: string format: date-time @@ -11913,6 +12020,11 @@ components: format: uri nullable: true maxLength: 2048 + ocw_topics: + type: array + items: + type: string + maxLength: 128 professional: type: boolean readOnly: true @@ -12000,6 +12112,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 next_start_date: type: string format: date-time @@ -12213,6 +12331,11 @@ components: format: uri nullable: true maxLength: 2048 + ocw_topics: + type: array + items: + type: string + maxLength: 128 professional: type: boolean readOnly: true @@ -12300,6 +12423,12 @@ components: nullable: true minLength: 1 maxLength: 2048 + ocw_topics: + type: array + items: + type: string + minLength: 1 + maxLength: 128 next_start_date: type: string format: date-time diff --git a/test_json/courses/16-01-unified-engineering-i-ii-iii-iv-fall-2005-spring-2006/data.json b/test_json/courses/16-01-unified-engineering-i-ii-iii-iv-fall-2005-spring-2006/data.json index 542e96fb1b..c676ac7f6e 100644 --- a/test_json/courses/16-01-unified-engineering-i-ii-iii-iv-fall-2005-spring-2006/data.json +++ b/test_json/courses/16-01-unified-engineering-i-ii-iii-iv-fall-2005-spring-2006/data.json @@ -85,12 +85,12 @@ "Exams with Solutions" ], "topics": [ - ["Engineering", "Aerospace Engineering", "Materials Selection"], - ["Engineering", "Aerospace Engineering", "Propulsion Systems"], - ["Science", "Physics", "Thermodynamics"], - ["Engineering", "Mechanical Engineering", "Fluid Mechanics"], - ["Engineering", "Aerospace Engineering"], - ["Business", "Project Management"] + ["Social Science", "Anthropology", "Ethnography"], + ["Humanities", "Philosophy", "Political Philosophy"] + ], + "mit_learn_topics": [ + ["Social Sciences", "Anthropology"], + ["Social Sciences", "Political Science"] ], "primary_course_number": "16.01", "extra_course_numbers": "16.02, 16.03, 16.04", From 1875d8f99d956300700a6304223a75c105d12da9 Mon Sep 17 00:00:00 2001 From: Doof Date: Wed, 25 Sep 2024 13:24:04 +0000 Subject: [PATCH 13/13] Release date for 0.19.4 --- RELEASE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index c9493d04b6..a899b8533d 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,7 +1,7 @@ Release Notes ============= -Version 0.19.4 +Version 0.19.4 (Released September 25, 2024) -------------- - new -> recently added (#1594)