From d7209f6d30bc3e56061f64218b7256a88eeaf58d Mon Sep 17 00:00:00 2001 From: Shankar Ambady Date: Thu, 19 Sep 2024 14:31:11 -0400 Subject: [PATCH 01/10] Codespace opensearch service fix (#1582) * testing codespace fix * testing fix --- docker-compose.codespaces.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.codespaces.yml b/docker-compose.codespaces.yml index 965f705a7e..8423ccce38 100644 --- a/docker-compose.codespaces.yml +++ b/docker-compose.codespaces.yml @@ -7,6 +7,8 @@ services: file: docker-compose.apps.yml service: web env_file: env/codespaces.env + depends_on: + - opensearch-node-mitopen-1 watch: extends: From 590bddd616071b81f43e573cdd6741e795e2bb2d Mon Sep 17 00:00:00 2001 From: Doof Date: Thu, 19 Sep 2024 19:25:01 +0000 Subject: [PATCH 02/10] Release 0.19.2 --- RELEASE.rst | 9 +++++++++ main/settings.py | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index 8c6a4bda8a..e8739530e8 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,15 @@ Release Notes ============= +Version 0.19.2 +-------------- + +- Codespace opensearch service fix (#1582) +- Remove learning_format, use delivery for search filter/facet (#1567) +- Signup popover owns url (#1576) +- [pre-commit.ci] pre-commit autoupdate (#1541) +- Remove an unused dependency (#1571) + Version 0.19.1 (Released September 18, 2024) -------------- diff --git a/main/settings.py b/main/settings.py index 3b3adc4d56..e3038ed7a8 100644 --- a/main/settings.py +++ b/main/settings.py @@ -33,7 +33,7 @@ from main.settings_pluggy import * # noqa: F403 from openapi.settings_spectacular import open_spectacular_settings -VERSION = "0.19.1" +VERSION = "0.19.2" log = logging.getLogger() From 1a1f5a8d4174e8da62cb372744108cc2a0f31478 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Fri, 20 Sep 2024 11:29:17 -0400 Subject: [PATCH 03/10] Fix extract_openedx_data and backpopulate_mit_edx_data commands to work with course/program datafiles (#1587) --- .../commands/backpopulate_mit_edx_data.py | 16 ++++++++++++---- .../management/commands/extract_openedx_data.py | 3 ++- learning_resources/tasks.py | 15 +++++++++++---- 3 files changed, 25 insertions(+), 9 deletions(-) diff --git a/learning_resources/management/commands/backpopulate_mit_edx_data.py b/learning_resources/management/commands/backpopulate_mit_edx_data.py index 3886cfe64b..77e183e185 100644 --- a/learning_resources/management/commands/backpopulate_mit_edx_data.py +++ b/learning_resources/management/commands/backpopulate_mit_edx_data.py @@ -22,9 +22,15 @@ def add_arguments(self, parser): help="Delete all existing records first", ) parser.add_argument( - "--api_datafile", - dest="api_datafile", - help="If provided, use this file as the source of API data", + "--api_course_datafile", + dest="api_course_datafile", + help="If provided, use this file as the source of course API data", + default=None, + ) + parser.add_argument( + "--api_program_datafile", + dest="api_program_datafile", + help="If provided, use this file as the source of program API data", default=None, ) super().add_arguments(parser) @@ -40,7 +46,9 @@ def handle(self, *args, **options): # noqa: ARG002 ): resource_delete_actions(learning_resource) else: - task = get_mit_edx_data.delay(options["api_datafile"]) + task = get_mit_edx_data.delay( + options["api_course_datafile"], options["api_program_datafile"] + ) self.stdout.write(f"Started task {task} to get MIT edX course data") self.stdout.write("Waiting on task...") start = now_in_utc() diff --git a/learning_resources/management/commands/extract_openedx_data.py b/learning_resources/management/commands/extract_openedx_data.py index b7f14d3fe1..84cde80f33 100644 --- a/learning_resources/management/commands/extract_openedx_data.py +++ b/learning_resources/management/commands/extract_openedx_data.py @@ -5,13 +5,14 @@ from django.core.management import BaseCommand -from learning_resources.etl import mit_edx, oll +from learning_resources.etl import mit_edx, mit_edx_programs, oll from learning_resources.etl.constants import ETLSource from main.utils import now_in_utc EXTRACTORS = { ETLSource.oll.name: oll.extract, ETLSource.mit_edx.name: mit_edx.extract, + f"{ETLSource.mit_edx.name}_programs": mit_edx_programs.extract, } diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py index 3657875b89..0fd1527a3e 100644 --- a/learning_resources/tasks.py +++ b/learning_resources/tasks.py @@ -48,15 +48,22 @@ def get_micromasters_data(): @app.task -def get_mit_edx_data(api_datafile=None) -> int: +def get_mit_edx_data( + api_course_datafile: str | None = None, api_program_datafile: str | None = None +) -> int: """Task to sync MIT edX data with the database Args: - api_datafile (str): If provided, use this file as the source of API data + api_course_datafile (str): If provided, use file as source of course API data Otherwise, the API is queried directly. + api_program_datafile (str): If provided, use file as source of program API data. + Otherwise, the API is queried directly. + + Returns: + int: The number of results that were fetched """ - courses = pipelines.mit_edx_courses_etl(api_datafile) - programs = pipelines.mit_edx_programs_etl(api_datafile) + courses = pipelines.mit_edx_courses_etl(api_course_datafile) + programs = pipelines.mit_edx_programs_etl(api_program_datafile) clear_search_cache() return len(courses) + len(programs) From c849ff9bb73e31283bc32681fa113f9ec1e04b7f Mon Sep 17 00:00:00 2001 From: Shankar Ambady Date: Mon, 23 Sep 2024 11:46:36 -0400 Subject: [PATCH 04/10] follow/unfollow popover (#1589) * adding success variant * adding working version * adding working version * fixing text and adding margin to buttons * lint fixes * lint fix * fixing test cases * closing popup before posting data * removing redundancy * removing redundancy * removing empty test * updating styles * updating styles * changes to match design * changes to match design --- .../FollowPopover/FollowPopover.tsx | 144 ++++++++++++++++++ .../SearchSubscriptionToggle.test.tsx | 7 +- .../SearchSubscriptionToggle.tsx | 64 ++++---- .../SignupPopover/SignupPopover.tsx | 2 + .../src/components/Button/Button.tsx | 22 +++ .../src/components/Popover/Popover.tsx | 1 + 6 files changed, 202 insertions(+), 38 deletions(-) create mode 100644 frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx diff --git a/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx b/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx new file mode 100644 index 0000000000..3d21398f11 --- /dev/null +++ b/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx @@ -0,0 +1,144 @@ +import React, { useMemo } from "react" +import { Popover, Typography, styled, Button } from "ol-components" +import type { PopoverProps } from "ol-components" +import { getSearchParamMap } from "@/common/utils" + +import { SignupPopover } from "../SignupPopover/SignupPopover" + +import { useUserMe } from "api/hooks/user" +import { SourceTypeEnum } from "api" +import { + useSearchSubscriptionCreate, + useSearchSubscriptionDelete, + useSearchSubscriptionList, +} from "api/hooks/searchSubscription" + +const StyledPopover = styled(Popover)({ + width: "300px", + maxWidth: "100vw", +}) +const HeaderText = styled(Typography)(({ theme }) => ({ + color: theme.custom.colors.darkGray2, + marginBottom: "8px", + ...theme.typography.subtitle2, +})) +const BodyText = styled(Typography)(({ theme }) => ({ + ...theme.typography.body2, + color: theme.custom.colors.silverGrayDark, + marginBottom: "16px", +})) + +const Footer = styled.div({ + display: "flex", + justifyContent: "end", + gap: "16px", +}) + +interface FollowPopoverProps + extends Pick { + itemName?: string + searchParams: URLSearchParams + sourceType: SourceTypeEnum +} + +const FollowPopover: React.FC = ({ + itemName, + searchParams, + sourceType, + ...props +}) => { + const { data: user } = useUserMe() + const subscribeParams: Record = useMemo(() => { + return { source_type: sourceType, ...getSearchParamMap(searchParams) } + }, [searchParams, sourceType]) + + const subscriptionDelete = useSearchSubscriptionDelete() + const subscriptionCreate = useSearchSubscriptionCreate() + const subscriptionList = useSearchSubscriptionList(subscribeParams, { + enabled: !!user?.is_authenticated, + }) + const unsubscribe = subscriptionDelete.mutate + const subscriptionId = subscriptionList.data?.[0]?.id + + const isSubscribed = !!subscriptionId + const handleFollowAction = async (): Promise => { + props.onClose() + if (!isSubscribed) { + await subscriptionCreate.mutateAsync({ + PercolateQuerySubscriptionRequestRequest: subscribeParams, + }) + } else { + unsubscribe(subscriptionId) + } + } + + if (user?.is_authenticated && subscriptionList.isLoading) return null + if (!user) return null + if (!user?.is_authenticated) { + return + } + + if (isSubscribed) { + return ( + + + You are following {itemName} + + + Unfollow to stop getting emails for new {itemName} courses. + +
+ + +
+ + ) + } + return ( + + Follow {itemName}? + + You will get an email when new courses are available. + +
+ + +
+ + ) +} + +export { FollowPopover } +export type { FollowPopoverProps } diff --git a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx index c3116e8a82..844e36e2eb 100644 --- a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx +++ b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx @@ -79,6 +79,8 @@ test.each(Object.values(SourceTypeEnum))( }) await user.click(subscribeButton) + const followButton = screen.getByTestId("action-follow") + await user.click(followButton) expect(makeRequest).toHaveBeenCalledWith("post", subscribeUrl, { source_type: sourceType, offered_by: ["ocw"], @@ -111,11 +113,8 @@ test.each(Object.values(SourceTypeEnum))( }) await user.click(subscribedButton) + const unsubscribeButton = screen.getByTestId("action-unfollow") - const menu = screen.getByRole("menu") - const unsubscribeButton = within(menu).getByRole("menuitem", { - name: "Unfollow", - }) await user.click(unsubscribeButton) expect(makeRequest).toHaveBeenCalledWith( diff --git a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx index b80f87e884..503b1f4cc4 100644 --- a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx +++ b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx @@ -2,15 +2,14 @@ import React, { useState, useMemo } from "react" import { getSearchParamMap } from "@/common/utils" import { useSearchSubscriptionCreate, - useSearchSubscriptionDelete, useSearchSubscriptionList, } from "api/hooks/searchSubscription" -import { Button, SimpleMenu, styled } from "ol-components" -import type { SimpleMenuItem } from "ol-components" -import { RiArrowDownSLine, RiMailLine } from "@remixicon/react" +import { Button, styled } from "ol-components" + +import { RiMailLine } from "@remixicon/react" import { useUserMe } from "api/hooks/user" import { SourceTypeEnum } from "api" -import { SignupPopover } from "../SignupPopover/SignupPopover" +import { FollowPopover } from "../FollowPopover/FollowPopover" const StyledButton = styled(Button)({ minWidth: "130px", @@ -23,6 +22,7 @@ type SearchSubscriptionToggleProps = { } const SearchSubscriptionToggle: React.FC = ({ + itemName, searchParams, sourceType, }) => { @@ -33,35 +33,17 @@ const SearchSubscriptionToggle: React.FC = ({ }, [searchParams, sourceType]) const { data: user } = useUserMe() - const subscriptionDelete = useSearchSubscriptionDelete() + const subscriptionCreate = useSearchSubscriptionCreate() const subscriptionList = useSearchSubscriptionList(subscribeParams, { enabled: !!user?.is_authenticated, }) - const unsubscribe = subscriptionDelete.mutate const subscriptionId = subscriptionList.data?.[0]?.id const isSubscribed = !!subscriptionId - const unsubscribeItems: SimpleMenuItem[] = useMemo(() => { - if (!subscriptionId) return [] - return [ - { - key: "unsubscribe", - label: "Unfollow", - onClick: () => unsubscribe(subscriptionId), - }, - ] - }, [unsubscribe, subscriptionId]) - const onFollowClick = async (event: React.MouseEvent) => { - if (user?.is_authenticated) { - await subscriptionCreate.mutateAsync({ - PercolateQuerySubscriptionRequestRequest: subscribeParams, - }) - } else { - setButtonEl(event.currentTarget) - } + setButtonEl(event.currentTarget) } if (user?.is_authenticated && subscriptionList.isLoading) return null @@ -69,14 +51,22 @@ const SearchSubscriptionToggle: React.FC = ({ if (isSubscribed) { return ( - }> - Following - - } - items={unsubscribeItems} - /> + <> + } + > + Following + + setButtonEl(null)} + /> + ) } @@ -90,7 +80,13 @@ const SearchSubscriptionToggle: React.FC = ({ > Follow - setButtonEl(null)} /> + setButtonEl(null)} + /> ) } diff --git a/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx b/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx index 64487d1b25..03100dabeb 100644 --- a/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx +++ b/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx @@ -11,10 +11,12 @@ const StyledPopover = styled(Popover)({ const HeaderText = styled(Typography)(({ theme }) => ({ color: theme.custom.colors.darkGray2, marginBottom: "8px", + ...theme.typography.subtitle2, })) const BodyText = styled(Typography)(({ theme }) => ({ color: theme.custom.colors.silverGrayDark, marginBottom: "16px", + ...theme.typography.body2, })) const Footer = styled.div({ diff --git a/frontends/ol-components/src/components/Button/Button.tsx b/frontends/ol-components/src/components/Button/Button.tsx index ac88f0db5d..641e264f18 100644 --- a/frontends/ol-components/src/components/Button/Button.tsx +++ b/frontends/ol-components/src/components/Button/Button.tsx @@ -12,6 +12,7 @@ type ButtonVariant = | "text" | "noBorder" | "inverted" + | "success" type ButtonSize = "small" | "medium" | "large" type ButtonEdge = "circular" | "rounded" | "none" @@ -127,6 +128,27 @@ const ButtonStyled = styled.button((props) => { borderColor: "currentcolor", borderStyle: "solid", }, + variant === "success" && { + backgroundColor: colors.darkGreen, + color: colors.white, + border: "none", + /* Shadow/04dp */ + boxShadow: + "0px 2px 4px 0px rgba(37, 38, 43, 0.10), 0px 3px 8px 0px rgba(37, 38, 43, 0.12)", + ":hover:not(:disabled)": { + backgroundColor: colors.darkGreen, + boxShadow: "none", + }, + ":disabled": { + backgroundColor: colors.silverGray, + boxShadow: "none", + }, + }, + hasBorder && { + backgroundColor: "transparent", + borderColor: "currentcolor", + borderStyle: "solid", + }, variant === "secondary" && { color: colors.red, ":hover:not(:disabled)": { diff --git a/frontends/ol-components/src/components/Popover/Popover.tsx b/frontends/ol-components/src/components/Popover/Popover.tsx index 48a332c227..39e96ff8af 100644 --- a/frontends/ol-components/src/components/Popover/Popover.tsx +++ b/frontends/ol-components/src/components/Popover/Popover.tsx @@ -128,6 +128,7 @@ const Arrow = styled("div")({ const Content = styled.div(({ theme }) => ({ padding: "16px", backgroundColor: theme.custom.colors.white, + borderRadius: "8px", boxShadow: "0px 2px 4px 0px rgba(37, 38, 43, 0.10), 0px 6px 24px 0px rgba(37, 38, 43, 0.24)", })) From 9e9fc0745b287058f06963d8324bd45935cdf19b Mon Sep 17 00:00:00 2001 From: Anastasia Beglova Date: Mon, 23 Sep 2024 12:18:31 -0400 Subject: [PATCH 05/10] Make search mode defaults settable env variables (#1590) --- app.json | 20 +++++++ frontends/mit-learn/jest.config.ts | 5 ++ .../SearchDisplay/SearchDisplay.tsx | 29 +++++++--- .../src/pages/SearchPage/SearchPage.test.tsx | 39 ++++++++----- frontends/mit-learn/webpack.config.js | 30 ++++++++++ .../ol-utilities/src/types/settings.d.ts | 5 ++ learning_resources_search/api.py | 30 +++++++++- learning_resources_search/api_test.py | 58 ++++++++++++++++++- learning_resources_search/serializers.py | 3 - main/settings.py | 13 +++++ openapi/specs/v1.yaml | 15 ----- 11 files changed, 204 insertions(+), 43 deletions(-) diff --git a/app.json b/app.json index e47e0e6f8e..4fe3feccae 100644 --- a/app.json +++ b/app.json @@ -71,6 +71,26 @@ "description": "The domain to set the CSRF cookie on", "required": false }, + "DEFAULT_SEARCH_MODE": { + "description": "Default search mode for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_SLOP": { + "description": "Default slop value for the search API and frontend. Only used for phrase queries.", + "required": false + }, + "DEFAULT_SEARCH_STALENESS_PENALTY": { + "description": "Default staleness penalty value for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF": { + "description": "Default minimum score cutoff value for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY": { + "description": "Default max incompleteness penalty value for the search API and frontend", + "required": false + }, "EDX_API_ACCESS_TOKEN_URL": { "description": "URL to retrieve a MITx access token", "required": false diff --git a/frontends/mit-learn/jest.config.ts b/frontends/mit-learn/jest.config.ts index c767e8583f..54925827a7 100644 --- a/frontends/mit-learn/jest.config.ts +++ b/frontends/mit-learn/jest.config.ts @@ -19,6 +19,11 @@ const config: Config.InitialOptions = { MITOL_API_BASE_URL: "https://api.test.learn.mit.edu", PUBLIC_URL: "", SITE_NAME: "MIT Learn", + DEFAULT_SEARCH_MODE: "phrase", + DEFAULT_SEARCH_SLOP: 6, + DEFAULT_SEARCH_STALENESS_PENALTY: 2.5, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: 0, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: 90, }, }, } diff --git a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx index d269fa71ae..7c27760057 100644 --- a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -172,6 +172,11 @@ const FacetStyles = styled.div` &.facets-expanded { max-height: 600px; + + &.admin-facet { + max-height: fit-content; + } + transition: max-height 0.4s ease-in; } @@ -515,6 +520,14 @@ interface SearchDisplayProps { filterHeadingEl: React.ElementType } +const { + DEFAULT_SEARCH_MODE, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, +} = APP_SETTINGS + const SearchDisplay: React.FC = ({ page, setPage, @@ -585,7 +598,7 @@ const SearchDisplay: React.FC = ({ const searchModeDropdown = ( setSearchParams((prev) => { const next = new URLSearchParams(prev) @@ -623,7 +636,7 @@ const SearchDisplay: React.FC = ({ return (
- {searchParams.get("search_mode") === "phrase" ? ( + {(!searchParams.get("search_mode") && + DEFAULT_SEARCH_MODE === "phrase") || + searchParams.get("search_mode") === "phrase" ? (
Slop @@ -674,7 +689,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("slop") ? Number(searchParams.get("slop")) - : 0 + : DEFAULT_SEARCH_SLOP } setSearchParams={setSearchParams} urlParam="slop" @@ -694,7 +709,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("min_score") ? Number(searchParams.get("min_score")) - : 0 + : DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF } setSearchParams={setSearchParams} urlParam="min_score" @@ -713,7 +728,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("max_incompleteness_penalty") ? Number(searchParams.get("max_incompleteness_penalty")) - : 0 + : DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY } setSearchParams={setSearchParams} urlParam="max_incompleteness_penalty" diff --git a/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx b/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx index 4a5fae0258..697d3f9ae5 100644 --- a/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx +++ b/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx @@ -306,6 +306,11 @@ describe("SearchPage", () => { setMockResponse.get(urls.userMe.get(), { is_learning_path_editor: true, }) + APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" + APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 + APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 + APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 renderWithProviders() await waitFor(() => { const adminFacetContainer = screen.getByText("Admin Options") @@ -321,6 +326,11 @@ describe("SearchPage", () => { }) test("admin users can set the search mode and slop", async () => { + APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" + APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 + APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 + APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 setMockApiResponses({ search: { count: 700, @@ -349,14 +359,24 @@ test("admin users can set the search mode and slop", async () => { user.click(adminFacetContainer) }) - let slopSlider = screen.queryByText("Slop") - expect(slopSlider).toBeNull() - - const searchModeDropdowns = await screen.findAllByText("best_fields") + const searchModeDropdowns = await screen.findAllByText("phrase") const searchModeDropdown = searchModeDropdowns[0] await user.click(searchModeDropdown) + const mostFieldsSelect = await screen.findByRole("option", { + name: "most_fields", + }) + + await user.click(mostFieldsSelect) + + expect(location.current.search).toBe("?search_mode=most_fields") + + const slopSlider = screen.queryByText("Slop") + expect(slopSlider).toBeNull() + + await user.click(searchModeDropdown) + const phraseSelect = await screen.findByRole("option", { name: "phrase", }) @@ -370,17 +390,6 @@ test("admin users can set the search mode and slop", async () => { }) await user.click(searchModeDropdown) - - const mostFieldsSelect = await screen.findByRole("option", { - name: "most_fields", - }) - - await user.click(mostFieldsSelect) - - expect(location.current.search).toBe("?search_mode=most_fields") - - slopSlider = screen.queryByText("Slop") - expect(slopSlider).toBeNull() }) describe("Search Page Tabs", () => { diff --git a/frontends/mit-learn/webpack.config.js b/frontends/mit-learn/webpack.config.js index 7e49fd3bbb..9d48b30bd0 100644 --- a/frontends/mit-learn/webpack.config.js +++ b/frontends/mit-learn/webpack.config.js @@ -45,6 +45,11 @@ const { CSRF_COOKIE_NAME, APPZI_URL, MITOL_NOINDEX, + DEFAULT_SEARCH_MODE, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, } = cleanEnv(process.env, { NODE_ENV: str({ choices: ["development", "production", "test"], @@ -124,6 +129,26 @@ const { desc: "Whether to include a noindex meta tag", default: true, }), + DEFAULT_SEARCH_SLOP: num({ + desc: "The default search slop", + default: 6, + }), + DEFAULT_SEARCH_STALENESS_PENALTY: num({ + desc: "The default search staleness penalty", + default: 2.5, + }), + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: num({ + desc: "The default search minimum score cutoff", + default: 0, + }), + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: num({ + desc: "The default search max incompleteness penalty", + default: 90, + }), + DEFAULT_SEARCH_MODE: str({ + desc: "The default search mode", + default: "phrase", + }), }) const MITOL_FEATURES_PREFIX = "FEATURE_" @@ -265,6 +290,11 @@ module.exports = (env, argv) => { MITOL_SUPPORT_EMAIL: JSON.stringify(MITOL_SUPPORT_EMAIL), PUBLIC_URL: JSON.stringify(PUBLIC_URL), CSRF_COOKIE_NAME: JSON.stringify(CSRF_COOKIE_NAME), + DEFAULT_SEARCH_MODE: JSON.stringify(DEFAULT_SEARCH_MODE), + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, }, }), ] diff --git a/frontends/ol-utilities/src/types/settings.d.ts b/frontends/ol-utilities/src/types/settings.d.ts index da51840dd7..553c75e0c8 100644 --- a/frontends/ol-utilities/src/types/settings.d.ts +++ b/frontends/ol-utilities/src/types/settings.d.ts @@ -22,5 +22,10 @@ export declare global { SITE_NAME: string MITOL_SUPPORT_EMAIL: string PUBLIC_URL: string + DEFAULT_SEARCH_MODE: string + DEFAULT_SEARCH_SLOP: number + DEFAULT_SEARCH_STALENESS_PENALTY: number + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: number + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: number } } diff --git a/learning_resources_search/api.py b/learning_resources_search/api.py index eb4582a581..f6e08a07d1 100644 --- a/learning_resources_search/api.py +++ b/learning_resources_search/api.py @@ -5,6 +5,7 @@ from collections import Counter from datetime import UTC, datetime +from django.conf import settings from opensearch_dsl import Search from opensearch_dsl.query import MoreLikeThis, Percolate from opensearchpy.exceptions import NotFoundError @@ -507,7 +508,19 @@ def adjust_original_query_for_percolate(query): Remove keys that are irrelevent when storing original queries for percolate uniqueness such as "limit" and "offset" """ - for key in ["limit", "offset", "sortby", "yearly_decay_percent", "dev_mode"]: + for key in [ + "limit", + "offset", + "sortby", + "yearly_decay_percent", + "dev_mode", + "use_dfs_query_then_fetch", + "max_incompleteness_penalty", + "min_score", + "search_mode", + "slop", + "use_dfs_query_then_fetch", + ]: query.pop(key, None) return order_params(query) @@ -684,6 +697,21 @@ def execute_learn_search(search_params): Returns: dict: The opensearch response dict """ + if search_params.get("endpoint") != CONTENT_FILE_TYPE: + if search_params.get("yearly_decay_percent") is None: + search_params["yearly_decay_percent"] = ( + settings.DEFAULT_SEARCH_STALENESS_PENALTY + ) + if search_params.get("search_mode") is None: + search_params["search_mode"] = settings.DEFAULT_SEARCH_MODE + if search_params.get("slop") is None: + search_params["slop"] = settings.DEFAULT_SEARCH_SLOP + if search_params.get("min_score") is None: + search_params["min_score"] = settings.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF + if search_params.get("max_incompleteness_penalty") is None: + search_params["max_incompleteness_penalty"] = ( + settings.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY + ) search = construct_search(search_params) return search.execute().to_dict() diff --git a/learning_resources_search/api_test.py b/learning_resources_search/api_test.py index 71282429f5..86b810113b 100644 --- a/learning_resources_search/api_test.py +++ b/learning_resources_search/api_test.py @@ -1051,6 +1051,10 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "offset": 1, "sortby": "-readable_id", "endpoint": LEARNING_RESOURCE, + "yearly_decay_percent": 0, + "max_incompleteness_penalty": 0, + "min_score": 0, + "search_mode": "best_fields", } query = { @@ -1079,6 +1083,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1090,6 +1095,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "topics.name" ], + "type": "best_fields", } }, } @@ -1104,6 +1110,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1117,6 +1124,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1132,6 +1140,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -1150,6 +1159,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -1168,6 +1178,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -1194,6 +1205,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1203,6 +1215,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "best_fields", } }, } @@ -1217,6 +1230,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1230,6 +1244,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1245,6 +1260,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -1263,6 +1279,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -1281,6 +1298,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -1450,8 +1468,12 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): ], ) def test_execute_learn_search_with_script_score( - mocker, opensearch, yearly_decay_percent, max_incompleteness_penalty + mocker, settings, opensearch, yearly_decay_percent, max_incompleteness_penalty ): + settings.DEFAULT_SEARCH_MODE = "phrase" + settings.DEFAULT_SEARCH_SLOP = 0 + settings.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + opensearch.conn.search.return_value = { "hits": {"total": {"value": 10, "relation": "eq"}} } @@ -1531,6 +1553,7 @@ def test_execute_learn_search_with_script_score( "course_feature", "video.transcript.english", ], + "type": "phrase", } }, { @@ -1542,6 +1565,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "topics.name" ], + "type": "phrase", } }, } @@ -1556,6 +1580,7 @@ def test_execute_learn_search_with_script_score( "departments.department_id", "departments.name", ], + "type": "phrase", } }, } @@ -1569,6 +1594,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "course.course_numbers.value" ], + "type": "phrase", } }, } @@ -1584,6 +1610,7 @@ def test_execute_learn_search_with_script_score( "runs.semester", "runs.level", ], + "type": "phrase", } }, } @@ -1602,6 +1629,7 @@ def test_execute_learn_search_with_script_score( "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "phrase", } }, } @@ -1620,6 +1648,7 @@ def test_execute_learn_search_with_script_score( "short_description.english^2", "content_feature_type", ], + "type": "phrase", } }, "score_mode": "avg", @@ -1646,6 +1675,7 @@ def test_execute_learn_search_with_script_score( "course_feature", "video.transcript.english", ], + "type": "phrase", } }, { @@ -1655,6 +1685,7 @@ def test_execute_learn_search_with_script_score( "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "phrase", } }, } @@ -1669,6 +1700,7 @@ def test_execute_learn_search_with_script_score( "departments.department_id", "departments.name", ], + "type": "phrase", } }, } @@ -1682,6 +1714,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "course.course_numbers.value" ], + "type": "phrase", } }, } @@ -1697,6 +1730,7 @@ def test_execute_learn_search_with_script_score( "runs.semester", "runs.level", ], + "type": "phrase", } }, } @@ -1715,6 +1749,7 @@ def test_execute_learn_search_with_script_score( "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "phrase", } }, } @@ -1733,6 +1768,7 @@ def test_execute_learn_search_with_script_score( "short_description.english^2", "content_feature_type", ], + "type": "phrase", } }, "score_mode": "avg", @@ -1898,11 +1934,15 @@ def test_execute_learn_search_with_script_score( ) -def test_execute_learn_search_with_min_score(mocker, opensearch): +def test_execute_learn_search_with_min_score(mocker, settings, opensearch): opensearch.conn.search.return_value = { "hits": {"total": {"value": 10, "relation": "eq"}} } + settings.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 0 + settings.DEFAULT_SEARCH_STALENESS_PENALTY = 0 + settings.DEFAULT_SEARCH_MODE = "best_fields" + search_params = { "aggregations": ["offered_by"], "q": "math", @@ -1943,6 +1983,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1954,6 +1995,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "topics.name" ], + "type": "best_fields", } }, } @@ -1968,6 +2010,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1981,6 +2024,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1996,6 +2040,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -2014,6 +2059,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -2032,6 +2078,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -2058,6 +2105,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -2067,6 +2115,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "best_fields", } }, } @@ -2081,6 +2130,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -2094,6 +2144,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -2109,6 +2160,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -2127,6 +2179,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -2145,6 +2198,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", diff --git a/learning_resources_search/serializers.py b/learning_resources_search/serializers.py index 54225e29fa..fd9bd3fdcf 100644 --- a/learning_resources_search/serializers.py +++ b/learning_resources_search/serializers.py @@ -331,7 +331,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=2.5, help_text=( "Relevance score penalty percent per year for for resources without " "upcoming runs. Only affects results if there is a search term." @@ -425,7 +424,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=0, help_text=( "Minimum score value a text query result needs to have to be displayed" ), @@ -435,7 +433,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=0, help_text=( "Maximum score penalty for incomplete OCW courses in percent. " "An OCW course with completeness = 0 will have this score penalty. " diff --git a/main/settings.py b/main/settings.py index 3b3adc4d56..38d644143c 100644 --- a/main/settings.py +++ b/main/settings.py @@ -766,3 +766,16 @@ def get_all_config_keys(): # Enable or disable search engine indexing MITOL_NOINDEX = get_bool("MITOL_NOINDEX", True) # noqa: FBT003 + +# Search defaults settings - adjustable throught the admin ui +DEFAULT_SEARCH_MODE = get_string(name="DEFAULT_SEARCH_MODE", default="phrase") +DEFAULT_SEARCH_SLOP = get_int(name="DEFAULT_SEARCH_SLOP", default=6) +DEFAULT_SEARCH_STALENESS_PENALTY = get_float( + name="DEFAULT_SEARCH_STALENESS_PENALTY", default=2.5 +) +DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = get_float( + name="DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF", default=0 +) +DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = get_float( + name="DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY", default=90 +) diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index af4d7a9bbf..ea686f47a4 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -2493,7 +2493,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -2506,7 +2505,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -2717,7 +2715,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -2976,7 +2973,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -2989,7 +2985,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -3200,7 +3195,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -3484,7 +3478,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -3497,7 +3490,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -3722,7 +3714,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -3983,7 +3974,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -3996,7 +3986,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -4221,7 +4210,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -10055,7 +10043,6 @@ components: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. certification: @@ -10136,7 +10123,6 @@ components: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed max_incompleteness_penalty: @@ -10145,7 +10131,6 @@ components: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. From ac06a68cf8fc5a015b6d4e188254f1c52d830253 Mon Sep 17 00:00:00 2001 From: Doof Date: Mon, 23 Sep 2024 16:20:38 +0000 Subject: [PATCH 06/10] Release date for 0.19.2 --- RELEASE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index e8739530e8..7655b99689 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,7 +1,7 @@ Release Notes ============= -Version 0.19.2 +Version 0.19.2 (Released September 23, 2024) -------------- - Codespace opensearch service fix (#1582) From dcb1a9ac65e529df0dbc6fb7350d9c4db03ab076 Mon Sep 17 00:00:00 2001 From: Doof Date: Mon, 23 Sep 2024 16:21:13 +0000 Subject: [PATCH 07/10] Release 0.19.3 --- RELEASE.rst | 7 +++++++ main/settings.py | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index 7655b99689..1f970b3ec9 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,13 @@ Release Notes ============= +Version 0.19.3 +-------------- + +- Make search mode defaults settable env variables (#1590) +- follow/unfollow popover (#1589) +- Fix extract_openedx_data and backpopulate_mit_edx_data commands to work with course/program datafiles (#1587) + Version 0.19.2 (Released September 23, 2024) -------------- diff --git a/main/settings.py b/main/settings.py index f73c0069d9..c4e3864445 100644 --- a/main/settings.py +++ b/main/settings.py @@ -33,7 +33,7 @@ from main.settings_pluggy import * # noqa: F403 from openapi.settings_spectacular import open_spectacular_settings -VERSION = "0.19.2" +VERSION = "0.19.3" log = logging.getLogger() From 9d67642926c566b55aab9969c93e04f931e65349 Mon Sep 17 00:00:00 2001 From: Doof Date: Mon, 23 Sep 2024 19:22:18 +0000 Subject: [PATCH 08/10] Release date for 0.19.3 --- RELEASE.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.rst b/RELEASE.rst index 1f970b3ec9..1ae708ba43 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,7 +1,7 @@ Release Notes ============= -Version 0.19.3 +Version 0.19.3 (Released September 23, 2024) -------------- - Make search mode defaults settable env variables (#1590) From 4bde7f1742afc38406581789287d759c9a918285 Mon Sep 17 00:00:00 2001 From: Matt Bertrand Date: Tue, 24 Sep 2024 08:48:46 -0400 Subject: [PATCH 09/10] Pace and format fields for learning resources (#1588) --- .pre-commit-config.yaml | 2 +- frontends/api/src/generated/v1/api.ts | 152 +++++++++++ .../test-utils/factories/learningResources.ts | 18 ++ learning_resources/constants.py | 14 + learning_resources/etl/constants.py | 2 +- learning_resources/etl/micromasters.py | 78 ++++-- learning_resources/etl/micromasters_test.py | 9 + learning_resources/etl/mitxonline.py | 52 +++- learning_resources/etl/mitxonline_test.py | 18 +- learning_resources/etl/ocw.py | 6 + learning_resources/etl/ocw_test.py | 6 + learning_resources/etl/oll.py | 6 + learning_resources/etl/oll_test.py | 8 + learning_resources/etl/openedx.py | 50 +++- learning_resources/etl/openedx_test.py | 16 ++ learning_resources/etl/prolearn.py | 10 +- learning_resources/etl/prolearn_test.py | 12 + learning_resources/etl/sloan.py | 54 +++- learning_resources/etl/sloan_test.py | 57 ++++ learning_resources/etl/xpro.py | 10 + learning_resources/etl/xpro_test.py | 14 + .../0068_learningresource_format_pace.py | 75 +++++ learning_resources/models.py | 40 +++ learning_resources/serializers.py | 36 +++ learning_resources/serializers_test.py | 9 + learning_resources_search/constants.py | 28 ++ main/models.py | 2 +- openapi/specs/v1.yaml | 256 ++++++++++++++++++ 28 files changed, 993 insertions(+), 47 deletions(-) create mode 100644 learning_resources/migrations/0068_learningresource_format_pace.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17d9ff63e6..0a30546482 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -74,7 +74,7 @@ repos: - ".*/generated/" additional_dependencies: ["gibberish-detector"] - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.6.5" + rev: "v0.6.6" hooks: - id: ruff-format - id: ruff diff --git a/frontends/api/src/generated/v1/api.ts b/frontends/api/src/generated/v1/api.ts index 983db187e9..ff24f6357f 100644 --- a/frontends/api/src/generated/v1/api.ts +++ b/frontends/api/src/generated/v1/api.ts @@ -686,6 +686,18 @@ export interface CourseResource { * @memberof CourseResource */ resource_category: string + /** + * + * @type {Array} + * @memberof CourseResource + */ + format: Array + /** + * + * @type {Array} + * @memberof CourseResource + */ + pace: Array /** * * @type {CourseResourceResourceTypeEnum} @@ -844,6 +856,62 @@ export const CourseResourceDeliveryInnerCodeEnum = { export type CourseResourceDeliveryInnerCodeEnum = (typeof CourseResourceDeliveryInnerCodeEnum)[keyof typeof CourseResourceDeliveryInnerCodeEnum] +/** + * + * @export + * @interface CourseResourceFormatInner + */ +export interface CourseResourceFormatInner { + /** + * + * @type {string} + * @memberof CourseResourceFormatInner + */ + code: CourseResourceFormatInnerCodeEnum + /** + * + * @type {string} + * @memberof CourseResourceFormatInner + */ + name: string +} + +export const CourseResourceFormatInnerCodeEnum = { + Synchronous: "synchronous", + Asynchronous: "asynchronous", +} as const + +export type CourseResourceFormatInnerCodeEnum = + (typeof CourseResourceFormatInnerCodeEnum)[keyof typeof CourseResourceFormatInnerCodeEnum] + +/** + * + * @export + * @interface CourseResourcePaceInner + */ +export interface CourseResourcePaceInner { + /** + * + * @type {string} + * @memberof CourseResourcePaceInner + */ + code: CourseResourcePaceInnerCodeEnum + /** + * + * @type {string} + * @memberof CourseResourcePaceInner + */ + name: string +} + +export const CourseResourcePaceInnerCodeEnum = { + SelfPaced: "self_paced", + InstructorPaced: "instructor_paced", +} as const + +export type CourseResourcePaceInnerCodeEnum = + (typeof CourseResourcePaceInnerCodeEnum)[keyof typeof CourseResourcePaceInnerCodeEnum] + /** * Serializer for course resources * @export @@ -1376,6 +1444,18 @@ export interface LearningPathResource { * @memberof LearningPathResource */ resource_category: string + /** + * + * @type {Array} + * @memberof LearningPathResource + */ + format: Array + /** + * + * @type {Array} + * @memberof LearningPathResource + */ + pace: Array /** * * @type {LearningPathResourceResourceTypeEnum} @@ -2100,6 +2180,18 @@ export interface LearningResourceRun { * @memberof LearningResourceRun */ delivery: Array + /** + * + * @type {Array} + * @memberof LearningResourceRun + */ + format: Array + /** + * + * @type {Array} + * @memberof LearningResourceRun + */ + pace: Array /** * * @type {string} @@ -4049,6 +4141,18 @@ export interface PodcastEpisodeResource { * @memberof PodcastEpisodeResource */ resource_category: string + /** + * + * @type {Array} + * @memberof PodcastEpisodeResource + */ + format: Array + /** + * + * @type {Array} + * @memberof PodcastEpisodeResource + */ + pace: Array /** * * @type {PodcastEpisodeResourceResourceTypeEnum} @@ -4395,6 +4499,18 @@ export interface PodcastResource { * @memberof PodcastResource */ resource_category: string + /** + * + * @type {Array} + * @memberof PodcastResource + */ + format: Array + /** + * + * @type {Array} + * @memberof PodcastResource + */ + pace: Array /** * * @type {PodcastResourceResourceTypeEnum} @@ -4961,6 +5077,18 @@ export interface ProgramResource { * @memberof ProgramResource */ resource_category: string + /** + * + * @type {Array} + * @memberof ProgramResource + */ + format: Array + /** + * + * @type {Array} + * @memberof ProgramResource + */ + pace: Array /** * * @type {ProgramResourceResourceTypeEnum} @@ -5786,6 +5914,18 @@ export interface VideoPlaylistResource { * @memberof VideoPlaylistResource */ resource_category: string + /** + * + * @type {Array} + * @memberof VideoPlaylistResource + */ + format: Array + /** + * + * @type {Array} + * @memberof VideoPlaylistResource + */ + pace: Array /** * * @type {VideoPlaylistResourceResourceTypeEnum} @@ -6120,6 +6260,18 @@ export interface VideoResource { * @memberof VideoResource */ resource_category: string + /** + * + * @type {Array} + * @memberof VideoResource + */ + format: Array + /** + * + * @type {Array} + * @memberof VideoResource + */ + pace: Array /** * * @type {VideoResourceResourceTypeEnum} diff --git a/frontends/api/src/test-utils/factories/learningResources.ts b/frontends/api/src/test-utils/factories/learningResources.ts index 8c11ddf195..f37c0092fb 100644 --- a/frontends/api/src/test-utils/factories/learningResources.ts +++ b/frontends/api/src/test-utils/factories/learningResources.ts @@ -30,6 +30,8 @@ import type { import { AvailabilityEnum, DeliveryEnum, + CourseResourcePaceInnerCodeEnum, + CourseResourceFormatInnerCodeEnum, ResourceTypeEnum, LearningResourceRunLevelInnerCodeEnum, PlatformEnum, @@ -186,6 +188,22 @@ const learningResourceRun: Factory = (overrides = {}) => { name: uniqueEnforcerWords.enforce(() => faker.lorem.words()), }, ], + pace: [ + { + code: faker.helpers.arrayElement( + Object.values(CourseResourcePaceInnerCodeEnum), + ), + name: uniqueEnforcerWords.enforce(() => faker.lorem.words()), + }, + ], + format: [ + { + code: faker.helpers.arrayElement( + Object.values(CourseResourceFormatInnerCodeEnum), + ), + name: uniqueEnforcerWords.enforce(() => faker.lorem.words()), + }, + ], level: [ { code: faker.helpers.arrayElement( diff --git a/learning_resources/constants.py b/learning_resources/constants.py index cdc609d824..41e84d6edb 100644 --- a/learning_resources/constants.py +++ b/learning_resources/constants.py @@ -287,3 +287,17 @@ class CertificationType(ExtendedEnum): professional = "Professional Certificate" completion = "Certificate of Completion" none = "No Certificate" + + +class Pace(ExtendedEnum): + """Enum for resource pace types""" + + self_paced = "Self-paced" + instructor_paced = "Instructor-paced" + + +class Format(ExtendedEnum): + """Enum for resource format types""" + + synchronous = "Synchronous" + asynchronous = "Asynchronous" diff --git a/learning_resources/etl/constants.py b/learning_resources/etl/constants.py index 319f85f245..fd1e7fcc6a 100644 --- a/learning_resources/etl/constants.py +++ b/learning_resources/etl/constants.py @@ -46,7 +46,7 @@ ) -class ETLSource(Enum): +class ETLSource(ExtendedEnum): """Enum of ETL sources""" micromasters = "micromasters" diff --git a/learning_resources/etl/micromasters.py b/learning_resources/etl/micromasters.py index b4bec1232e..6bebb30d35 100644 --- a/learning_resources/etl/micromasters.py +++ b/learning_resources/etl/micromasters.py @@ -8,12 +8,13 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, OfferedBy, PlatformType, ) from learning_resources.etl.constants import COMMON_HEADERS, ETLSource -from learning_resources.models import LearningResource +from learning_resources.models import LearningResource, default_pace OFFERED_BY = {"code": OfferedBy.mitx.name} READABLE_ID_PREFIX = "micromasters-program-" @@ -46,6 +47,27 @@ def _is_published(course_id: str) -> bool: return False +def _get_course_pace(course_id: str) -> list[str]: + """ + Determine the pace of the course by id + + Args: + course_id (str): the course id + + Returns: + list[str]: the pace of the course as a list of strings + + """ + existing_course = LearningResource.objects.filter( + readable_id=course_id, + resource_type=LearningResourceType.course.name, + published=True, + ).first() + if existing_course: + return existing_course.pace + return default_pace() + + def _transform_image(micromasters_data: dict) -> dict: """ Transform an image into our normalized data structure @@ -65,7 +87,36 @@ def transform(programs_data): programs = [] for program in programs_data: url = program.get("programpage_url") + # need positioning of courses by course_id for course data + courses = [ + { + "readable_id": course["edx_key"], + "platform": PlatformType.edx.name, + "offered_by": OFFERED_BY, + "published": _is_published(course["edx_key"]), + "pace": _get_course_pace(course["edx_key"]), + "runs": [ + { + "run_id": run["edx_course_key"], + } + for run in course["course_runs"] + if run.get("edx_course_key", None) + ], + } + for course in sorted( + program["courses"], + key=lambda course: course["position_in_program"], + ) + ] if url and DEDP not in url: + pace = sorted( + { + pace + for course in courses + for pace in course["pace"] + if course["published"] + } + ) programs.append( { "readable_id": f"{READABLE_ID_PREFIX}{program['id']}", @@ -91,30 +142,15 @@ def transform(programs_data): "end_date": program["end_date"], "enrollment_start": program["enrollment_start"], "availability": Availability.dated.name, + "pace": pace, + "format": [Format.asynchronous.name], } ], "topics": program["topics"], "availability": Availability.dated.name, - # only need positioning of courses by course_id for course data - "courses": [ - { - "readable_id": course["edx_key"], - "platform": PlatformType.edx.name, - "offered_by": OFFERED_BY, - "published": _is_published(course["edx_key"]), - "runs": [ - { - "run_id": run["edx_course_key"], - } - for run in course["course_runs"] - if run.get("edx_course_key", None) - ], - } - for course in sorted( - program["courses"], - key=lambda course: course["position_in_program"], - ) - ], + "pace": pace, + "format": [Format.asynchronous.name], + "courses": courses, } ) return programs diff --git a/learning_resources/etl/micromasters_test.py b/learning_resources/etl/micromasters_test.py index ede2c8df77..343d61260c 100644 --- a/learning_resources/etl/micromasters_test.py +++ b/learning_resources/etl/micromasters_test.py @@ -6,7 +6,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, + Pace, PlatformType, ) from learning_resources.etl import micromasters @@ -111,6 +113,7 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): readable_id="1", resource_type=LearningResourceType.course.name, etl_source=ETLSource.mit_edx.name, + pace=[Pace.instructor_paced.name], ) if missing_url: mock_micromasters_data[0]["programpage_url"] = None @@ -129,6 +132,8 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "certification": True, "certification_type": CertificationType.micromasters.name, "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], "courses": [ { "readable_id": "1", @@ -140,6 +145,7 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "run_id": "course_key_1", } ], + "pace": [Pace.instructor_paced.name], }, { "readable_id": "2", @@ -147,6 +153,7 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "offered_by": micromasters.OFFERED_BY, "published": False, "runs": [], + "pace": [Pace.self_paced.name], }, ], "runs": [ @@ -162,6 +169,8 @@ def test_micromasters_transform(mock_micromasters_data, missing_url): "end_date": None, "enrollment_start": "2019-09-29T20:13:26.367297Z", "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } ], "topics": [{"name": "program"}, {"name": "first"}], diff --git a/learning_resources/etl/mitxonline.py b/learning_resources/etl/mitxonline.py index 7495bd9f12..296174d960 100644 --- a/learning_resources/etl/mitxonline.py +++ b/learning_resources/etl/mitxonline.py @@ -12,8 +12,10 @@ from learning_resources.constants import ( CertificationType, + Format, LearningResourceType, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -237,6 +239,12 @@ def _transform_run(course_run: dict, course: dict) -> dict: if parse_page_attribute(course, "page_url") else RunStatus.archived.value, "availability": course.get("availability"), + "format": [Format.asynchronous.name], + "pace": [ + Pace.self_paced.name + if course_run.get("is_self_paced", False) + else Pace.instructor_paced.name + ], } @@ -283,6 +291,8 @@ def _transform_course(course): "url": parse_page_attribute(course, "page_url", is_url=True), "description": clean_data(parse_page_attribute(course, "description")), "availability": course.get("availability"), + "format": [Format.asynchronous.name], + "pace": sorted({pace for run in runs for pace in run["pace"]}), } @@ -320,12 +330,30 @@ def _fetch_courses_by_ids(course_ids): return [] -def transform_programs(programs): - """Transform the MITX Online catalog data""" - # normalize the MITx Online data +def transform_programs(programs: list[dict]) -> list[dict]: + """ + Transform the MITX Online catalog data - return [ - { + Args: + programs (list of dict): the MITX Online programs data + + Returns: + list of dict: the transformed programs data + + """ + # normalize the MITx Online data + for program in programs: + courses = transform_courses( + [ + course + for course in _fetch_courses_by_ids(program["courses"]) + if not re.search(EXCLUDE_REGEX, course["title"], re.IGNORECASE) + ] + ) + pace = sorted( + {course_pace for course in courses for course_pace in course["pace"]} + ) + yield { "readable_id": program["readable_id"], "title": program["title"], "offered_by": OFFERED_BY, @@ -346,6 +374,8 @@ def transform_programs(programs): "published": bool( parse_page_attribute(program, "page_url") ), # a program is only considered published if it has a page url + "format": [Format.asynchronous.name], + "pace": pace, "runs": [ { "run_id": program["readable_id"], @@ -371,15 +401,9 @@ def transform_programs(programs): if parse_page_attribute(program, "page_url") else RunStatus.archived.value, "availability": program.get("availability"), + "format": [Format.asynchronous.name], + "pace": pace, } ], - "courses": transform_courses( - [ - course - for course in _fetch_courses_by_ids(program["courses"]) - if not re.search(EXCLUDE_REGEX, course["title"], re.IGNORECASE) - ] - ), + "courses": courses, } - for program in programs - ] diff --git a/learning_resources/etl/mitxonline_test.py b/learning_resources/etl/mitxonline_test.py index 637cc81209..a7364d393c 100644 --- a/learning_resources/etl/mitxonline_test.py +++ b/learning_resources/etl/mitxonline_test.py @@ -11,7 +11,9 @@ from learning_resources.constants import ( CertificationType, + Format, LearningResourceType, + Pace, PlatformType, RunStatus, ) @@ -150,6 +152,8 @@ def test_mitxonline_transform_programs( "url": parse_page_attribute(program_data, "page_url", is_url=True), "availability": program_data["availability"], "topics": transform_topics(program_data["topics"], OFFERED_BY["code"]), + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], "runs": [ { "run_id": program_data["readable_id"], @@ -171,6 +175,8 @@ def test_mitxonline_transform_programs( if parse_page_attribute(program_data, "page_url") else RunStatus.archived.value, "availability": program_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } ], "courses": [ @@ -205,6 +211,8 @@ def test_mitxonline_transform_programs( "certification_type": CertificationType.completion.name, "url": parse_page_attribute(course_data, "page_url", is_url=True), "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], "topics": transform_topics( course_data["topics"], OFFERED_BY["code"] ), @@ -250,6 +258,8 @@ def test_mitxonline_transform_programs( if parse_page_attribute(course_data, "page_url") else RunStatus.archived.value, "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } for course_run_data in course_data["courseruns"] ], @@ -379,6 +389,8 @@ def test_mitxonline_transform_courses(settings, mock_mitxonline_courses_data): if parse_page_attribute(course_data, "page_url") else RunStatus.archived.value, "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } for course_run_data in course_data["courseruns"] ], @@ -394,6 +406,8 @@ def test_mitxonline_transform_courses(settings, mock_mitxonline_courses_data): ] }, "availability": course_data["availability"], + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } for course_data in mock_mitxonline_courses_data["results"] if "PROCTORED EXAM" not in course_data["title"] @@ -463,7 +477,9 @@ def test_program_run_start_date_value( # noqa: PLR0913 mock_mitxonline_programs_data["results"][0]["start_date"] = start_dt mock_mitxonline_programs_data["results"][0]["enrollment_start"] = enrollment_dt - transformed_programs = transform_programs(mock_mitxonline_programs_data["results"]) + transformed_programs = list( + transform_programs(mock_mitxonline_programs_data["results"]) + ) assert transformed_programs[0]["runs"][0]["start_date"] == _parse_datetime( expected_dt diff --git a/learning_resources/etl/ocw.py b/learning_resources/etl/ocw.py index f11310b8e2..d776f63c82 100644 --- a/learning_resources/etl/ocw.py +++ b/learning_resources/etl/ocw.py @@ -20,9 +20,11 @@ CONTENT_TYPE_VIDEO, VALID_TEXT_FILE_TYPES, Availability, + Format, LearningResourceDelivery, LearningResourceType, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -285,6 +287,8 @@ def transform_run(course_data: dict) -> dict: "url": course_data["url"], "availability": Availability.anytime.name, "delivery": parse_delivery(course_data), + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } @@ -368,6 +372,8 @@ def transform_course(course_data: dict) -> dict: "availability": Availability.anytime.name, "delivery": parse_delivery(course_data), "license_cc": True, + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } diff --git a/learning_resources/etl/ocw_test.py b/learning_resources/etl/ocw_test.py index 5d2ba82828..d54b562ce7 100644 --- a/learning_resources/etl/ocw_test.py +++ b/learning_resources/etl/ocw_test.py @@ -12,7 +12,9 @@ from learning_resources.constants import ( DEPARTMENTS, Availability, + Format, LearningResourceDelivery, + Pace, ) from learning_resources.etl.constants import CourseNumberType, ETLSource from learning_resources.etl.ocw import ( @@ -248,6 +250,10 @@ def test_transform_course( # noqa: PLR0913 assert transformed_json["runs"][0]["delivery"] == expected_delivery assert transformed_json["runs"][0]["availability"] == Availability.anytime.name assert transformed_json["availability"] == Availability.anytime.name + assert transformed_json["runs"][0]["pace"] == [Pace.self_paced.name] + assert transformed_json["runs"][0]["format"] == [Format.asynchronous.name] + assert transformed_json["pace"] == [Pace.self_paced.name] + assert transformed_json["format"] == [Format.asynchronous.name] assert transformed_json["description"] == clean_data( course_json["course_description_html"] ) diff --git a/learning_resources/etl/oll.py b/learning_resources/etl/oll.py index 5c720cbafe..96bd8436b3 100644 --- a/learning_resources/etl/oll.py +++ b/learning_resources/etl/oll.py @@ -13,7 +13,9 @@ from learning_resources.constants import ( Availability, + Format, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -143,6 +145,8 @@ def transform_run(course_data: dict) -> list[dict]: ], "status": RunStatus.archived.value, "availability": Availability.anytime.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } ] @@ -183,6 +187,8 @@ def transform_course(course_data: dict) -> dict: "prices": [Decimal(0.00)], "etl_source": ETLSource.oll.name, "availability": Availability.anytime.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } diff --git a/learning_resources/etl/oll_test.py b/learning_resources/etl/oll_test.py index f0eebf8ea6..91e9e05c7d 100644 --- a/learning_resources/etl/oll_test.py +++ b/learning_resources/etl/oll_test.py @@ -73,6 +73,8 @@ def test_oll_transform(mocker, oll_course_data): "year": 2022, "status": "Archived", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } ], "image": { @@ -82,6 +84,8 @@ def test_oll_transform(mocker, oll_course_data): "prices": [0.00], "etl_source": "oll", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } assert results[2] == { "title": "Competency-Based Education", @@ -125,6 +129,8 @@ def test_oll_transform(mocker, oll_course_data): "year": 2019, "status": "Archived", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } ], "image": { @@ -134,4 +140,6 @@ def test_oll_transform(mocker, oll_course_data): "prices": [0.00], "etl_source": "oll", "availability": "anytime", + "pace": ["self_paced"], + "format": ["asynchronous"], } diff --git a/learning_resources/etl/openedx.py b/learning_resources/etl/openedx.py index c7cc98b532..18a1ce118c 100644 --- a/learning_resources/etl/openedx.py +++ b/learning_resources/etl/openedx.py @@ -18,7 +18,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, + Pace, PlatformType, RunStatus, ) @@ -290,7 +292,7 @@ def _parse_course_dates(program, date_field): [ run[date_field] for run in course["course_runs"] - if run["status"] == "published" and run[date_field] + if _get_run_published(run) and run[date_field] ] ) return dates @@ -312,7 +314,7 @@ def _get_course_price(course): for run in sorted( course["course_runs"], key=lambda x: x["start"], reverse=False ): - if run["status"] == "published": + if _get_run_published(run): return min( [ Decimal(seat["price"]) @@ -356,6 +358,8 @@ def _transform_course_run(config, course_run, course_last_modified, marketing_ur "image": _transform_course_image(course_run.get("image")), "status": course_run.get("availability"), "availability": _get_run_availability(course_run).name, + "format": [Format.asynchronous.name], + "pace": [course_run.get("pacing_type") or Pace.self_paced.name], "url": marketing_url or "{}{}/course/".format(config.alt_url, course_run.get("key")), "prices": sorted( @@ -401,6 +405,25 @@ def _parse_program_instructors_topics(program): ) +def _parse_course_pace(runs: list[dict]) -> list[str]: + """ + Parse the pace of a course based on its runs + + Args: + runs (list of dict): the runs data + + Returns: + str: the pace of the course or programe + """ + pace = sorted( + {run["pacing_type"] for run in runs if run and _get_run_published(run)} + ) + if len(pace) == 0: + # Archived courses are considered self-paced + pace = [Pace.self_paced.name] + return pace + + def _transform_program_course(config: OpenEdxConfiguration, course: dict) -> dict: """ Transform a program's course dict to a normalized data structure @@ -419,6 +442,7 @@ def _transform_program_course(config: OpenEdxConfiguration, course: dict) -> dic "platform": config.platform, "resource_type": LearningResourceType.course.name, "offered_by": {"code": config.offered_by}, + "pace": _parse_course_pace(course.get("course_runs", [])), } @@ -458,6 +482,14 @@ def _transform_program_run( "prices": [_sum_course_prices(program)], "instructors": program.pop("instructors", []), "availability": _get_program_availability(program), + "format": [Format.asynchronous.name], + "pace": sorted( + { + pace + for course in program.get("courses", []) + for pace in _parse_course_pace(course.get("course_runs", [])) + } + ), } @@ -507,6 +539,8 @@ def _transform_course(config: OpenEdxConfiguration, course: dict) -> dict: if has_certification else CertificationType.none.name, "availability": _get_course_availability(course), + "format": [Format.asynchronous.name], + "pace": _parse_course_pace(course.get("course_runs", [])), } @@ -526,6 +560,11 @@ def _transform_program(config: OpenEdxConfiguration, program: dict) -> dict: image = _transform_program_image(program) instructors, topics = _parse_program_instructors_topics(program) program["instructors"] = instructors + courses = [ + _transform_program_course(config, course) + for course in program.get("courses", []) + ] + paces = sorted({pace for course in courses for pace in course["pace"]}) runs = [_transform_program_run(program, last_modified, image)] has_certification = parse_certification(config.offered_by, runs) return { @@ -549,10 +588,9 @@ def _transform_program(config: OpenEdxConfiguration, program: dict) -> dict: if has_certification else CertificationType.none.name, "availability": runs[0]["availability"], - "courses": [ - _transform_program_course(config, course) - for course in program.get("courses", []) - ], + "format": [Format.asynchronous.name], + "pace": paces, + "courses": courses, } diff --git a/learning_resources/etl/openedx_test.py b/learning_resources/etl/openedx_test.py index 3503cc15c4..3ae662fbe7 100644 --- a/learning_resources/etl/openedx_test.py +++ b/learning_resources/etl/openedx_test.py @@ -10,8 +10,10 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -212,6 +214,13 @@ def test_transform_course( # noqa: PLR0913 "last_modified": any_instance_of(datetime), "topics": [{"name": "Data Analysis & Statistics"}], "url": "http://localhost/fake-alt-url/this_course", + "format": [Format.asynchronous.name], + "pace": [Pace.instructor_paced.name] + if has_runs + and not is_run_deleted + and is_run_published + and is_run_enrollable + else [Pace.self_paced.name], "published": is_run_published and is_run_enrollable and not is_run_deleted @@ -249,6 +258,8 @@ def test_transform_course( # noqa: PLR0913 "year": 2019, "published": is_run_enrollable and is_run_published, "availability": Availability.dated.name, + "format": [Format.asynchronous.name], + "pace": [Pace.instructor_paced.name], } ] ), @@ -422,6 +433,8 @@ def test_transform_program( "certification": False, "certification_type": CertificationType.none.name, "availability": Availability.anytime.name, + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], "runs": ( [ { @@ -450,6 +463,8 @@ def test_transform_program( "url": extracted[0]["marketing_url"], "published": True, "availability": Availability.anytime.name, + "format": [Format.asynchronous.name], + "pace": [Pace.self_paced.name], } ] ), @@ -460,6 +475,7 @@ def test_transform_program( "platform": openedx_program_config.platform, "readable_id": f"MITx+6.002.{i}x", "resource_type": LearningResourceType.course.name, + "pace": [Pace.self_paced.name], } for i in range(1, 4) ], diff --git a/learning_resources/etl/prolearn.py b/learning_resources/etl/prolearn.py index de00d3b9d3..022aa6cc62 100644 --- a/learning_resources/etl/prolearn.py +++ b/learning_resources/etl/prolearn.py @@ -9,7 +9,7 @@ import requests from django.conf import settings -from learning_resources.constants import Availability, CertificationType +from learning_resources.constants import Availability, CertificationType, Format, Pace from learning_resources.etl.constants import ETLSource from learning_resources.etl.utils import transform_delivery, transform_topics from learning_resources.models import LearningResourceOfferor, LearningResourcePlatform @@ -281,10 +281,14 @@ def transform_programs(programs: list[dict]) -> list[dict]: } ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for course_id in sorted(program["field_related_courses_programs"]) ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } unique_program = unique_programs.setdefault( transformed_program["url"], transformed_program @@ -321,6 +325,8 @@ def _transform_runs(resource: dict) -> list[dict]: "url": parse_url(resource), "delivery": transform_delivery(resource["format_name"]), "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } ) return runs @@ -361,6 +367,8 @@ def _transform_course( "runs": runs, "unique_field": UNIQUE_FIELD, "availability": Availability.dated.name, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } return None diff --git a/learning_resources/etl/prolearn_test.py b/learning_resources/etl/prolearn_test.py index 00eb8fc267..9390b73a48 100644 --- a/learning_resources/etl/prolearn_test.py +++ b/learning_resources/etl/prolearn_test.py @@ -10,8 +10,10 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceDelivery, OfferedBy, + Pace, PlatformType, ) from learning_resources.etl.constants import ETLSource @@ -177,6 +179,8 @@ def test_prolearn_transform_programs(mock_csail_programs_data): ), "availability": Availability.dated.name, "delivery": transform_delivery(program["format_name"]), + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for (start_val, end_val) in zip( program["start_value"], program["end_value"] @@ -201,10 +205,14 @@ def test_prolearn_transform_programs(mock_csail_programs_data): } ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for course_id in sorted(program["field_related_courses_programs"]) ], "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for program in extracted_data[1:] ] @@ -254,6 +262,8 @@ def test_prolearn_transform_courses(mock_mitpe_courses_data): ), "availability": Availability.dated.name, "delivery": transform_delivery(course["format_name"]), + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for (start_val, end_val) in zip( course["start_value"], course["end_value"] @@ -261,6 +271,8 @@ def test_prolearn_transform_courses(mock_mitpe_courses_data): ], "course": {"course_numbers": []}, "unique_field": UNIQUE_FIELD, + "pace": [Pace.instructor_paced.name], + "format": [Format.asynchronous.name], } for course in extracted_data[2:] ] diff --git a/learning_resources/etl/sloan.py b/learning_resources/etl/sloan.py index 3697c5690d..a58b4e311e 100644 --- a/learning_resources/etl/sloan.py +++ b/learning_resources/etl/sloan.py @@ -12,7 +12,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, OfferedBy, + Pace, PlatformType, RunStatus, ) @@ -21,6 +23,7 @@ transform_delivery, transform_topics, ) +from learning_resources.models import default_format log = logging.getLogger(__name__) @@ -124,6 +127,49 @@ def parse_availability(run_data: dict) -> str: return Availability.dated.name +def parse_pace(run_data: dict) -> str: + """ + Parse pace from run data + + Args: + run_data (list): the run data + + Returns: + str: the pace + """ + if run_data and ( + run_data.get("Delivery") == "Online" + and run_data.get("Format") == "Asynchronous (On-Demand)" + ): + return Pace.self_paced.name + return Pace.instructor_paced.name + + +def parse_format(run_data: dict) -> str: + """ + Parse format from run data + + Args: + run_data (list): the run data + + Returns: + str: the format code + """ + if run_data: + delivery = run_data.get("Delivery") + if delivery == "In Person": + return [Format.synchronous.name] + elif delivery == "Blended": + return [Format.synchronous.name, Format.asynchronous.name] + else: + return ( + [Format.asynchronous.name] + if "Asynchronous" in (run_data.get("Format") or "") + else [Format.synchronous.name] + ) + return default_format() + + def extract(): """ Extract Sloan Executive Education data @@ -184,6 +230,8 @@ def transform_run(run_data, course_data): "published": True, "prices": [run_data["Price"]], "instructors": [{"full_name": name.strip()} for name in faculty_names], + "pace": [parse_pace(run_data)], + "format": parse_format(run_data), } @@ -205,6 +253,8 @@ def transform_course(course_data: dict, runs_data: dict) -> dict: format_delivery = list( {transform_delivery(run["Delivery"])[0] for run in course_runs_data} ) + runs = [transform_run(run, course_data) for run in course_runs_data] + transformed_course = { "readable_id": course_data["Course_Id"], "title": course_data["Title"], @@ -223,8 +273,10 @@ def transform_course(course_data: dict, runs_data: dict) -> dict: "course": { "course_numbers": [], }, - "runs": [transform_run(run, course_data) for run in course_runs_data], + "runs": runs, "continuing_ed_credits": course_runs_data[0]["Continuing_Ed_Credits"], + "pace": sorted({pace for run in runs for pace in run["pace"]}), + "format": sorted({run_format for run in runs for run_format in run["format"]}), } return transformed_course if transformed_course.get("url") else None diff --git a/learning_resources/etl/sloan_test.py b/learning_resources/etl/sloan_test.py index 9cb6a6cfa6..85c472d15e 100644 --- a/learning_resources/etl/sloan_test.py +++ b/learning_resources/etl/sloan_test.py @@ -7,13 +7,17 @@ from learning_resources.constants import ( Availability, + Format, + Pace, RunStatus, ) from learning_resources.etl.sloan import ( extract, parse_availability, parse_datetime, + parse_format, parse_image, + parse_pace, transform_course, transform_delivery, transform_run, @@ -124,6 +128,8 @@ def test_transform_run( "published": True, "prices": [run_data["Price"]], "instructors": [{"full_name": name.strip()} for name in faculty_names], + "pace": [Pace.instructor_paced.name], + "format": [Format.synchronous.name], } @@ -147,6 +153,10 @@ def test_transform_course(mock_sloan_courses_data, mock_sloan_runs_data): assert transformed["runs"][0]["availability"] == parse_availability( course_runs_data[0] ) + assert transformed["pace"] == [Pace.instructor_paced.name] + assert transformed["format"] == [Format.asynchronous.name, Format.synchronous.name] + assert transformed["runs"][0]["pace"] == [Pace.instructor_paced.name] + assert transformed["runs"][0]["format"] == [Format.asynchronous.name] assert transformed["image"] == parse_image(course_data) assert ( transformed["continuing_ed_credits"] @@ -171,6 +181,8 @@ def test_transform_course(mock_sloan_courses_data, mock_sloan_runs_data): "course", "runs", "continuing_ed_credits", + "pace", + "format", ] ) @@ -223,3 +235,48 @@ def test_enabled_flag(mock_sloan_api_setting, settings): """Extract should return empty lists if the SEE_API_ENABLED flag is False""" settings.SEE_API_ENABLED = False assert extract() == ([], []) + + +@pytest.mark.parametrize( + ("delivery", "run_format", "pace"), + [ + ("Online", "Synchronous", Pace.instructor_paced.name), + ("Online", "Asynchronous (On-Demand)", Pace.self_paced.name), + ("Online", "Asynchronous (Date based)", Pace.instructor_paced.name), + ("In Person", "Asynchronous (On-Demand)", Pace.instructor_paced.name), + ], +) +def test_parse_pace(delivery, run_format, pace): + """Test that the pace is parsed correctly""" + run_data = { + "Format": run_format, + "Delivery": delivery, + } + assert parse_pace(run_data) == pace + assert parse_pace(None) == Pace.instructor_paced.name + + +@pytest.mark.parametrize( + ("delivery", "run_format", "expected_format"), + [ + ("In Person", "Asynchronous (On-Demand)", [Format.synchronous.name]), + ( + "Blended", + "Asynchronous (On-Demand)", + [Format.synchronous.name, Format.asynchronous.name], + ), + ("Online", "Synchronous", [Format.synchronous.name]), + ("Online", "Asynchronous (On-Demand)", [Format.asynchronous.name]), + ("Online", "Asynchronous (Date based)", [Format.asynchronous.name]), + ("Online", None, [Format.synchronous.name]), + (None, None, [Format.synchronous.name]), + ], +) +def test_parse_format(delivery, run_format, expected_format): + """Test that the format is parsed correctly""" + run_data = { + "Format": run_format, + "Delivery": delivery, + } + assert parse_format(run_data) == expected_format + assert parse_format(None) == [Format.asynchronous.name] diff --git a/learning_resources/etl/xpro.py b/learning_resources/etl/xpro.py index 2dc7f08214..c6f3f4b225 100644 --- a/learning_resources/etl/xpro.py +++ b/learning_resources/etl/xpro.py @@ -10,8 +10,10 @@ from learning_resources.constants import ( CertificationType, + Format, LearningResourceType, OfferedBy, + Pace, PlatformType, ) from learning_resources.etl.constants import ETLSource @@ -117,6 +119,8 @@ def _transform_run(course_run: dict, course: dict) -> dict: ], "availability": course["availability"], "delivery": transform_delivery(course.get("format")), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } @@ -158,6 +162,8 @@ def _transform_learning_resource_course(course): "certification_type": CertificationType.professional.name, "availability": course["availability"], "continuing_ed_credits": course["credits"], + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } @@ -216,6 +222,8 @@ def transform_programs(programs): ], "delivery": transform_delivery(program.get("format")), "availability": program["availability"], + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } ], "courses": transform_courses(program["courses"]), @@ -223,6 +231,8 @@ def transform_programs(programs): "certification_type": CertificationType.professional.name, "availability": program["availability"], "continuing_ed_credits": program["credits"], + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for program in programs ] diff --git a/learning_resources/etl/xpro_test.py b/learning_resources/etl/xpro_test.py index 0f1954a5c2..7acb9e8226 100644 --- a/learning_resources/etl/xpro_test.py +++ b/learning_resources/etl/xpro_test.py @@ -10,7 +10,9 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceType, + Pace, PlatformType, ) from learning_resources.etl import xpro @@ -112,6 +114,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): "resource_type": LearningResourceType.program.name, "delivery": transform_delivery(program_data.get("format")), "continuing_ed_credits": program_data.get("credits"), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], "runs": [ { "run_id": program_data["readable_id"], @@ -131,6 +135,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): "description": program_data["description"], "delivery": transform_delivery(program_data.get("format")), "availability": Availability.dated.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } ], "courses": [ @@ -153,6 +159,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): "topics": parse_topics(course_data), "resource_type": LearningResourceType.course.name, "continuing_ed_credits": course_data.get("credits"), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], "runs": [ { "run_id": course_run_data["courseware_id"], @@ -173,6 +181,8 @@ def test_xpro_transform_programs(mock_xpro_programs_data): ], "delivery": transform_delivery(course_data.get("format")), "availability": Availability.dated.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for course_run_data in course_data["courseruns"] ], @@ -245,6 +255,8 @@ def test_xpro_transform_courses(mock_xpro_courses_data): ], "delivery": transform_delivery(course_data.get("format")), "availability": Availability.dated.name, + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for course_run_data in course_data["courseruns"] ], @@ -262,6 +274,8 @@ def test_xpro_transform_courses(mock_xpro_courses_data): "certification": True, "certification_type": CertificationType.professional.name, "continuing_ed_credits": course_data.get("credits"), + "pace": [Pace.self_paced.name], + "format": [Format.asynchronous.name], } for course_data in mock_xpro_courses_data ] diff --git a/learning_resources/migrations/0068_learningresource_format_pace.py b/learning_resources/migrations/0068_learningresource_format_pace.py new file mode 100644 index 0000000000..6929be78d1 --- /dev/null +++ b/learning_resources/migrations/0068_learningresource_format_pace.py @@ -0,0 +1,75 @@ +# Generated by Django 4.2.16 on 2024-09-23 14:18 + +import django.contrib.postgres.fields +from django.db import migrations, models + +import learning_resources.models + + +class Migration(migrations.Migration): + dependencies = [ + ("learning_resources", "0067_ocw_delivery_online_only"), + ] + + operations = [ + migrations.AddField( + model_name="learningresource", + name="format", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("synchronous", "Synchronous"), + ("asynchronous", "Asynchronous"), + ], + max_length=24, + ), + default=learning_resources.models.default_format, + size=None, + ), + ), + migrations.AddField( + model_name="learningresource", + name="pace", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("self_paced", "Self-paced"), + ("instructor_paced", "Instructor-paced"), + ], + max_length=24, + ), + default=learning_resources.models.default_pace, + size=None, + ), + ), + migrations.AddField( + model_name="learningresourcerun", + name="format", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("synchronous", "Synchronous"), + ("asynchronous", "Asynchronous"), + ], + max_length=24, + ), + default=learning_resources.models.default_format, + size=None, + ), + ), + migrations.AddField( + model_name="learningresourcerun", + name="pace", + field=django.contrib.postgres.fields.ArrayField( + base_field=models.CharField( + choices=[ + ("self_paced", "Self-paced"), + ("instructor_paced", "Instructor-paced"), + ], + max_length=24, + ), + default=learning_resources.models.default_pace, + size=None, + ), + ), + ] diff --git a/learning_resources/models.py b/learning_resources/models.py index 069214b7a6..4b74e723cd 100644 --- a/learning_resources/models.py +++ b/learning_resources/models.py @@ -16,9 +16,11 @@ from learning_resources.constants import ( Availability, CertificationType, + Format, LearningResourceDelivery, LearningResourceRelationTypes, LearningResourceType, + Pace, PrivacyLevel, ) from main.models import TimestampedModel, TimestampedModelQuerySet @@ -29,6 +31,16 @@ def default_delivery(): return [LearningResourceDelivery.online.name] +def default_pace(): + """Return the default pace as a list""" + return [Pace.self_paced.name] + + +def default_format(): + """Return the default format as a list""" + return [Format.asynchronous.name] + + class LearningResourcePlatform(TimestampedModel): """Platforms for all learning resources""" @@ -428,6 +440,20 @@ class LearningResource(TimestampedModel): continuing_ed_credits = models.DecimalField( max_digits=5, decimal_places=2, null=True, blank=True ) + pace = ArrayField( + models.CharField( + max_length=24, + choices=Pace.as_tuple(), + ), + default=default_pace, + ) + format = ArrayField( + models.CharField( + max_length=24, + choices=Format.as_tuple(), + ), + default=default_format, + ) @property def audience(self) -> str | None: @@ -565,6 +591,20 @@ class LearningResourceRun(TimestampedModel): null=True, choices=Availability.as_tuple(), ) + pace = ArrayField( + models.CharField( + max_length=24, + choices=Pace.as_tuple(), + ), + default=default_pace, + ) + format = ArrayField( + models.CharField( + max_length=24, + choices=Format.as_tuple(), + ), + default=default_format, + ) def __str__(self): return f"LearningResourceRun platform={self.learning_resource.platform} run_id={self.run_id}" # noqa: E501 diff --git a/learning_resources/serializers.py b/learning_resources/serializers.py index 64b76380eb..c52a40ebc6 100644 --- a/learning_resources/serializers.py +++ b/learning_resources/serializers.py @@ -15,9 +15,11 @@ from learning_resources.constants import ( LEARNING_MATERIAL_RESOURCE_CATEGORY, CertificationType, + Format, LearningResourceDelivery, LearningResourceType, LevelType, + Pace, ) from learning_resources.etl.loaders import update_index from main.serializers import COMMON_IGNORED_FIELDS, WriteableSerializerMethodField @@ -236,6 +238,36 @@ def to_representation(self, value): return {"code": value, "name": LearningResourceDelivery[value].value} +@extend_schema_field( + { + "type": "object", + "properties": { + "code": {"enum": Format.names()}, + "name": {"type": "string"}, + }, + "required": ["code", "name"], + } +) +class FormatSerializer(serializers.Field): + def to_representation(self, value): + return {"code": value, "name": Format[value].value} + + +@extend_schema_field( + { + "type": "object", + "properties": { + "code": {"enum": Pace.names()}, + "name": {"type": "string"}, + }, + "required": ["code", "name"], + } +) +class PaceSerializer(serializers.Field): + def to_representation(self, value): + return {"code": value, "name": Pace[value].value} + + class LearningResourceRunSerializer(serializers.ModelSerializer): """Serializer for the LearningResourceRun model""" @@ -248,6 +280,8 @@ class LearningResourceRunSerializer(serializers.ModelSerializer): delivery = serializers.ListField( child=LearningResourceDeliverySerializer(), read_only=True ) + format = serializers.ListField(child=FormatSerializer(), read_only=True) + pace = serializers.ListField(child=PaceSerializer(), read_only=True) class Meta: model = models.LearningResourceRun @@ -415,6 +449,8 @@ class LearningResourceBaseSerializer(serializers.ModelSerializer, WriteableTopic ) free = serializers.SerializerMethodField() resource_category = serializers.SerializerMethodField() + format = serializers.ListField(child=FormatSerializer(), read_only=True) + pace = serializers.ListField(child=PaceSerializer(), read_only=True) def get_resource_category(self, instance) -> str: """Return the resource category of the resource""" diff --git a/learning_resources/serializers_test.py b/learning_resources/serializers_test.py index 3ca30f06dc..bc68711f96 100644 --- a/learning_resources/serializers_test.py +++ b/learning_resources/serializers_test.py @@ -12,9 +12,11 @@ from learning_resources.constants import ( LEARNING_MATERIAL_RESOURCE_CATEGORY, CertificationType, + Format, LearningResourceDelivery, LearningResourceRelationTypes, LearningResourceType, + Pace, PlatformType, ) from learning_resources.models import ContentFile, LearningResource @@ -262,6 +264,13 @@ def test_learning_resource_serializer( # noqa: PLR0913 {"code": lr_delivery, "name": LearningResourceDelivery[lr_delivery].value} for lr_delivery in resource.delivery ], + "format": [ + {"code": lr_format, "name": Format[lr_format].value} + for lr_format in resource.format + ], + "pace": [ + {"code": lr_pace, "name": Pace[lr_pace].value} for lr_pace in resource.pace + ], "next_start_date": resource.next_start_date, "availability": resource.availability, "completeness": 1.0, diff --git a/learning_resources_search/constants.py b/learning_resources_search/constants.py index b0b1d53051..a4208aaf4e 100644 --- a/learning_resources_search/constants.py +++ b/learning_resources_search/constants.py @@ -121,6 +121,20 @@ class FilterConfig: "name": {"type": "keyword"}, }, }, + "pace": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, + "format": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, "readable_id": {"type": "keyword"}, "title": ENGLISH_TEXT_FIELD_WITH_SUGGEST, "description": ENGLISH_TEXT_FIELD_WITH_SUGGEST, @@ -231,6 +245,20 @@ class FilterConfig: "name": {"type": "keyword"}, }, }, + "pace": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, + "format": { + "type": "nested", + "properties": { + "code": {"type": "keyword"}, + "name": {"type": "keyword"}, + }, + }, "semester": {"type": "keyword"}, "year": {"type": "keyword"}, "start_date": {"type": "date"}, diff --git a/main/models.py b/main/models.py index 686cea755a..4bafc925ff 100644 --- a/main/models.py +++ b/main/models.py @@ -18,7 +18,7 @@ def update(self, **kwargs): Automatically update updated_on timestamp when .update(). This is because .update() does not go through .save(), thus will not auto_now, because it happens on the database level without loading objects into memory. - """ # noqa: D402, E501 + """ # noqa: E501 if "updated_on" not in kwargs: kwargs["updated_on"] = now_in_utc() return super().update(**kwargs) diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index ea686f47a4..2f6057ec71 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -7956,6 +7956,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/CourseResourceResourceTypeEnum' @@ -8023,11 +8053,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional @@ -8398,6 +8430,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/LearningPathResourceResourceTypeEnum' @@ -8463,12 +8525,14 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path - learning_path_parents - offered_by + - pace - platform - prices - readable_id @@ -8964,6 +9028,36 @@ components: - code - name readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true run_id: type: string maxLength: 128 @@ -9040,10 +9134,12 @@ components: - $ref: '#/components/schemas/NullEnum' required: - delivery + - format - id - image - instructors - level + - pace - run_id - title LearningResourceRunRequest: @@ -10405,6 +10501,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/PodcastEpisodeResourceResourceTypeEnum' @@ -10471,11 +10597,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - podcast_episode - prices @@ -10687,6 +10815,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/PodcastResourceResourceTypeEnum' @@ -10753,11 +10911,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - podcast - prices @@ -11099,6 +11259,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/ProgramResourceResourceTypeEnum' @@ -11165,11 +11355,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional @@ -11653,6 +11845,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/VideoPlaylistResourceResourceTypeEnum' @@ -11719,11 +11941,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional @@ -11921,6 +12145,36 @@ components: type: string description: Return the resource category of the resource readOnly: true + format: + type: array + items: + type: object + properties: + code: + enum: + - synchronous + - asynchronous + name: + type: string + required: + - code + - name + readOnly: true + pace: + type: array + items: + type: object + properties: + code: + enum: + - self_paced + - instructor_paced + name: + type: string + required: + - code + - name + readOnly: true resource_type: allOf: - $ref: '#/components/schemas/VideoResourceResourceTypeEnum' @@ -11987,11 +12241,13 @@ components: - course_feature - delivery - departments + - format - free - id - image - learning_path_parents - offered_by + - pace - platform - prices - professional From 743b12e3353cf24971f6ff47660e4c4da4a5f389 Mon Sep 17 00:00:00 2001 From: Jon Kafton <939376+jonkafton@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:42:24 +0200 Subject: [PATCH 10/10] Migrate search settings #5527 --- env/frontend.env | 6 ++++ frontends/main/Dockerfile.web | 6 ++++ .../app-pages/SearchPage/SearchPage.test.tsx | 20 ++++++------- .../FollowPopover/FollowPopover.tsx | 0 .../SearchDisplay/SearchDisplay.tsx | 22 +++++++++----- frontends/mit-learn/webpack.config.js | 30 ------------------- 6 files changed, 36 insertions(+), 48 deletions(-) rename frontends/{mit-learn => main}/src/page-components/FollowPopover/FollowPopover.tsx (100%) diff --git a/env/frontend.env b/env/frontend.env index 1781f81d48..b6d13aa462 100644 --- a/env/frontend.env +++ b/env/frontend.env @@ -10,3 +10,9 @@ NEXT_PUBLIC_MITOL_SUPPORT_EMAIL=${MITOL_SUPPORT_EMAIL} NEXT_PUBLIC_SITE_NAME="MIT Learn" NEXT_PUBLIC_MITOL_AXIOS_WITH_CREDENTIALS=true + +NEXT_PUBLIC_DEFAULT_SEARCH_MODE=phrase +NEXT_PUBLIC_DEFAULT_SEARCH_SLOP=6 +NEXT_PUBLIC_DEFAULT_SEARCH_STALENESS_PENALTY=2.5 +NEXT_PUBLIC_DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF=0 +NEXT_PUBLIC_DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY=90 diff --git a/frontends/main/Dockerfile.web b/frontends/main/Dockerfile.web index 0035f1f362..eb53e51cbc 100644 --- a/frontends/main/Dockerfile.web +++ b/frontends/main/Dockerfile.web @@ -79,6 +79,12 @@ ENV NEXT_PUBLIC_MITOL_AXIOS_WITH_CREDENTIALS=$NEXT_PUBLIC_MITOL_AXIOS_WITH_CREDE ARG NEXT_PUBLIC_CSRF_COOKIE_NAME ENV NEXT_PUBLIC_CSRF_COOKIE_NAME=$NEXT_PUBLIC_CSRF_COOKIE_NAME +ENV NEXT_PUBLIC_DEFAULT_SEARCH_MODE="phrase" +ENV NEXT_PUBLIC_DEFAULT_SEARCH_SLOP="6" +ENV NEXT_PUBLIC_DEFAULT_SEARCH_STALENESS_PENALTY="2.5" +ENV NEXT_PUBLIC_DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF="0" +ENV NEXT_PUBLIC_DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY="90" + RUN yarn build EXPOSE 3000 diff --git a/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx b/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx index 697d3f9ae5..16186eebaa 100644 --- a/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx +++ b/frontends/main/src/app-pages/SearchPage/SearchPage.test.tsx @@ -306,11 +306,11 @@ describe("SearchPage", () => { setMockResponse.get(urls.userMe.get(), { is_learning_path_editor: true, }) - APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" - APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 - APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 - APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 - APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MODE = "phrase" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_SLOP = "6" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_STALENESS_PENALTY = "2.5" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = "0" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = "90" renderWithProviders() await waitFor(() => { const adminFacetContainer = screen.getByText("Admin Options") @@ -326,11 +326,11 @@ describe("SearchPage", () => { }) test("admin users can set the search mode and slop", async () => { - APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" - APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 - APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 - APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 - APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MODE = "phrase" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_SLOP = "6" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_STALENESS_PENALTY = "2.5" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = "0" + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = "90" setMockApiResponses({ search: { count: 700, diff --git a/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx b/frontends/main/src/page-components/FollowPopover/FollowPopover.tsx similarity index 100% rename from frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx rename to frontends/main/src/page-components/FollowPopover/FollowPopover.tsx diff --git a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx index 69f36b605f..98f6c45797 100644 --- a/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/main/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -519,14 +519,6 @@ interface SearchDisplayProps { filterHeadingEl: React.ElementType } -const { - DEFAULT_SEARCH_MODE, - DEFAULT_SEARCH_SLOP, - DEFAULT_SEARCH_STALENESS_PENALTY, - DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, - DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, -} = APP_SETTINGS - const SearchDisplay: React.FC = ({ page, setPage, @@ -543,6 +535,20 @@ const SearchDisplay: React.FC = ({ resultsHeadingEl, filterHeadingEl, }) => { + const DEFAULT_SEARCH_MODE = process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MODE + const DEFAULT_SEARCH_SLOP = parseFloat( + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_SLOP!, + ) + const DEFAULT_SEARCH_STALENESS_PENALTY = parseFloat( + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_STALENESS_PENALTY!, + ) + const DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = parseFloat( + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF!, + ) + const DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = parseFloat( + process.env.NEXT_PUBLIC_DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY!, + ) + const [searchParams] = useSearchParams() const [expandAdminOptions, setExpandAdminOptions] = useState(false) const scrollHook = useRef(null) diff --git a/frontends/mit-learn/webpack.config.js b/frontends/mit-learn/webpack.config.js index 9d48b30bd0..7e49fd3bbb 100644 --- a/frontends/mit-learn/webpack.config.js +++ b/frontends/mit-learn/webpack.config.js @@ -45,11 +45,6 @@ const { CSRF_COOKIE_NAME, APPZI_URL, MITOL_NOINDEX, - DEFAULT_SEARCH_MODE, - DEFAULT_SEARCH_SLOP, - DEFAULT_SEARCH_STALENESS_PENALTY, - DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, - DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, } = cleanEnv(process.env, { NODE_ENV: str({ choices: ["development", "production", "test"], @@ -129,26 +124,6 @@ const { desc: "Whether to include a noindex meta tag", default: true, }), - DEFAULT_SEARCH_SLOP: num({ - desc: "The default search slop", - default: 6, - }), - DEFAULT_SEARCH_STALENESS_PENALTY: num({ - desc: "The default search staleness penalty", - default: 2.5, - }), - DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: num({ - desc: "The default search minimum score cutoff", - default: 0, - }), - DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: num({ - desc: "The default search max incompleteness penalty", - default: 90, - }), - DEFAULT_SEARCH_MODE: str({ - desc: "The default search mode", - default: "phrase", - }), }) const MITOL_FEATURES_PREFIX = "FEATURE_" @@ -290,11 +265,6 @@ module.exports = (env, argv) => { MITOL_SUPPORT_EMAIL: JSON.stringify(MITOL_SUPPORT_EMAIL), PUBLIC_URL: JSON.stringify(PUBLIC_URL), CSRF_COOKIE_NAME: JSON.stringify(CSRF_COOKIE_NAME), - DEFAULT_SEARCH_MODE: JSON.stringify(DEFAULT_SEARCH_MODE), - DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, - DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, - DEFAULT_SEARCH_SLOP, - DEFAULT_SEARCH_STALENESS_PENALTY, }, }), ]