diff --git a/RELEASE.rst b/RELEASE.rst index 7655b99689..1f970b3ec9 100644 --- a/RELEASE.rst +++ b/RELEASE.rst @@ -1,6 +1,13 @@ Release Notes ============= +Version 0.19.3 +-------------- + +- Make search mode defaults settable env variables (#1590) +- follow/unfollow popover (#1589) +- Fix extract_openedx_data and backpopulate_mit_edx_data commands to work with course/program datafiles (#1587) + Version 0.19.2 (Released September 23, 2024) -------------- diff --git a/app.json b/app.json index e47e0e6f8e..4fe3feccae 100644 --- a/app.json +++ b/app.json @@ -71,6 +71,26 @@ "description": "The domain to set the CSRF cookie on", "required": false }, + "DEFAULT_SEARCH_MODE": { + "description": "Default search mode for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_SLOP": { + "description": "Default slop value for the search API and frontend. Only used for phrase queries.", + "required": false + }, + "DEFAULT_SEARCH_STALENESS_PENALTY": { + "description": "Default staleness penalty value for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF": { + "description": "Default minimum score cutoff value for the search API and frontend", + "required": false + }, + "DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY": { + "description": "Default max incompleteness penalty value for the search API and frontend", + "required": false + }, "EDX_API_ACCESS_TOKEN_URL": { "description": "URL to retrieve a MITx access token", "required": false diff --git a/frontends/mit-learn/jest.config.ts b/frontends/mit-learn/jest.config.ts index c767e8583f..54925827a7 100644 --- a/frontends/mit-learn/jest.config.ts +++ b/frontends/mit-learn/jest.config.ts @@ -19,6 +19,11 @@ const config: Config.InitialOptions = { MITOL_API_BASE_URL: "https://api.test.learn.mit.edu", PUBLIC_URL: "", SITE_NAME: "MIT Learn", + DEFAULT_SEARCH_MODE: "phrase", + DEFAULT_SEARCH_SLOP: 6, + DEFAULT_SEARCH_STALENESS_PENALTY: 2.5, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: 0, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: 90, }, }, } diff --git a/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx b/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx new file mode 100644 index 0000000000..3d21398f11 --- /dev/null +++ b/frontends/mit-learn/src/page-components/FollowPopover/FollowPopover.tsx @@ -0,0 +1,144 @@ +import React, { useMemo } from "react" +import { Popover, Typography, styled, Button } from "ol-components" +import type { PopoverProps } from "ol-components" +import { getSearchParamMap } from "@/common/utils" + +import { SignupPopover } from "../SignupPopover/SignupPopover" + +import { useUserMe } from "api/hooks/user" +import { SourceTypeEnum } from "api" +import { + useSearchSubscriptionCreate, + useSearchSubscriptionDelete, + useSearchSubscriptionList, +} from "api/hooks/searchSubscription" + +const StyledPopover = styled(Popover)({ + width: "300px", + maxWidth: "100vw", +}) +const HeaderText = styled(Typography)(({ theme }) => ({ + color: theme.custom.colors.darkGray2, + marginBottom: "8px", + ...theme.typography.subtitle2, +})) +const BodyText = styled(Typography)(({ theme }) => ({ + ...theme.typography.body2, + color: theme.custom.colors.silverGrayDark, + marginBottom: "16px", +})) + +const Footer = styled.div({ + display: "flex", + justifyContent: "end", + gap: "16px", +}) + +interface FollowPopoverProps + extends Pick { + itemName?: string + searchParams: URLSearchParams + sourceType: SourceTypeEnum +} + +const FollowPopover: React.FC = ({ + itemName, + searchParams, + sourceType, + ...props +}) => { + const { data: user } = useUserMe() + const subscribeParams: Record = useMemo(() => { + return { source_type: sourceType, ...getSearchParamMap(searchParams) } + }, [searchParams, sourceType]) + + const subscriptionDelete = useSearchSubscriptionDelete() + const subscriptionCreate = useSearchSubscriptionCreate() + const subscriptionList = useSearchSubscriptionList(subscribeParams, { + enabled: !!user?.is_authenticated, + }) + const unsubscribe = subscriptionDelete.mutate + const subscriptionId = subscriptionList.data?.[0]?.id + + const isSubscribed = !!subscriptionId + const handleFollowAction = async (): Promise => { + props.onClose() + if (!isSubscribed) { + await subscriptionCreate.mutateAsync({ + PercolateQuerySubscriptionRequestRequest: subscribeParams, + }) + } else { + unsubscribe(subscriptionId) + } + } + + if (user?.is_authenticated && subscriptionList.isLoading) return null + if (!user) return null + if (!user?.is_authenticated) { + return + } + + if (isSubscribed) { + return ( + + + You are following {itemName} + + + Unfollow to stop getting emails for new {itemName} courses. + +
+ + +
+ + ) + } + return ( + + Follow {itemName}? + + You will get an email when new courses are available. + +
+ + +
+ + ) +} + +export { FollowPopover } +export type { FollowPopoverProps } diff --git a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx index d269fa71ae..7c27760057 100644 --- a/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx +++ b/frontends/mit-learn/src/page-components/SearchDisplay/SearchDisplay.tsx @@ -172,6 +172,11 @@ const FacetStyles = styled.div` &.facets-expanded { max-height: 600px; + + &.admin-facet { + max-height: fit-content; + } + transition: max-height 0.4s ease-in; } @@ -515,6 +520,14 @@ interface SearchDisplayProps { filterHeadingEl: React.ElementType } +const { + DEFAULT_SEARCH_MODE, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, +} = APP_SETTINGS + const SearchDisplay: React.FC = ({ page, setPage, @@ -585,7 +598,7 @@ const SearchDisplay: React.FC = ({ const searchModeDropdown = ( setSearchParams((prev) => { const next = new URLSearchParams(prev) @@ -623,7 +636,7 @@ const SearchDisplay: React.FC = ({ return (
- {searchParams.get("search_mode") === "phrase" ? ( + {(!searchParams.get("search_mode") && + DEFAULT_SEARCH_MODE === "phrase") || + searchParams.get("search_mode") === "phrase" ? (
Slop @@ -674,7 +689,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("slop") ? Number(searchParams.get("slop")) - : 0 + : DEFAULT_SEARCH_SLOP } setSearchParams={setSearchParams} urlParam="slop" @@ -694,7 +709,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("min_score") ? Number(searchParams.get("min_score")) - : 0 + : DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF } setSearchParams={setSearchParams} urlParam="min_score" @@ -713,7 +728,7 @@ const SearchDisplay: React.FC = ({ currentValue={ searchParams.get("max_incompleteness_penalty") ? Number(searchParams.get("max_incompleteness_penalty")) - : 0 + : DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY } setSearchParams={setSearchParams} urlParam="max_incompleteness_penalty" diff --git a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx index c3116e8a82..844e36e2eb 100644 --- a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx +++ b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.test.tsx @@ -79,6 +79,8 @@ test.each(Object.values(SourceTypeEnum))( }) await user.click(subscribeButton) + const followButton = screen.getByTestId("action-follow") + await user.click(followButton) expect(makeRequest).toHaveBeenCalledWith("post", subscribeUrl, { source_type: sourceType, offered_by: ["ocw"], @@ -111,11 +113,8 @@ test.each(Object.values(SourceTypeEnum))( }) await user.click(subscribedButton) + const unsubscribeButton = screen.getByTestId("action-unfollow") - const menu = screen.getByRole("menu") - const unsubscribeButton = within(menu).getByRole("menuitem", { - name: "Unfollow", - }) await user.click(unsubscribeButton) expect(makeRequest).toHaveBeenCalledWith( diff --git a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx index b80f87e884..503b1f4cc4 100644 --- a/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx +++ b/frontends/mit-learn/src/page-components/SearchSubscriptionToggle/SearchSubscriptionToggle.tsx @@ -2,15 +2,14 @@ import React, { useState, useMemo } from "react" import { getSearchParamMap } from "@/common/utils" import { useSearchSubscriptionCreate, - useSearchSubscriptionDelete, useSearchSubscriptionList, } from "api/hooks/searchSubscription" -import { Button, SimpleMenu, styled } from "ol-components" -import type { SimpleMenuItem } from "ol-components" -import { RiArrowDownSLine, RiMailLine } from "@remixicon/react" +import { Button, styled } from "ol-components" + +import { RiMailLine } from "@remixicon/react" import { useUserMe } from "api/hooks/user" import { SourceTypeEnum } from "api" -import { SignupPopover } from "../SignupPopover/SignupPopover" +import { FollowPopover } from "../FollowPopover/FollowPopover" const StyledButton = styled(Button)({ minWidth: "130px", @@ -23,6 +22,7 @@ type SearchSubscriptionToggleProps = { } const SearchSubscriptionToggle: React.FC = ({ + itemName, searchParams, sourceType, }) => { @@ -33,35 +33,17 @@ const SearchSubscriptionToggle: React.FC = ({ }, [searchParams, sourceType]) const { data: user } = useUserMe() - const subscriptionDelete = useSearchSubscriptionDelete() + const subscriptionCreate = useSearchSubscriptionCreate() const subscriptionList = useSearchSubscriptionList(subscribeParams, { enabled: !!user?.is_authenticated, }) - const unsubscribe = subscriptionDelete.mutate const subscriptionId = subscriptionList.data?.[0]?.id const isSubscribed = !!subscriptionId - const unsubscribeItems: SimpleMenuItem[] = useMemo(() => { - if (!subscriptionId) return [] - return [ - { - key: "unsubscribe", - label: "Unfollow", - onClick: () => unsubscribe(subscriptionId), - }, - ] - }, [unsubscribe, subscriptionId]) - const onFollowClick = async (event: React.MouseEvent) => { - if (user?.is_authenticated) { - await subscriptionCreate.mutateAsync({ - PercolateQuerySubscriptionRequestRequest: subscribeParams, - }) - } else { - setButtonEl(event.currentTarget) - } + setButtonEl(event.currentTarget) } if (user?.is_authenticated && subscriptionList.isLoading) return null @@ -69,14 +51,22 @@ const SearchSubscriptionToggle: React.FC = ({ if (isSubscribed) { return ( - }> - Following - - } - items={unsubscribeItems} - /> + <> + } + > + Following + + setButtonEl(null)} + /> + ) } @@ -90,7 +80,13 @@ const SearchSubscriptionToggle: React.FC = ({ > Follow - setButtonEl(null)} /> + setButtonEl(null)} + /> ) } diff --git a/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx b/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx index 64487d1b25..03100dabeb 100644 --- a/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx +++ b/frontends/mit-learn/src/page-components/SignupPopover/SignupPopover.tsx @@ -11,10 +11,12 @@ const StyledPopover = styled(Popover)({ const HeaderText = styled(Typography)(({ theme }) => ({ color: theme.custom.colors.darkGray2, marginBottom: "8px", + ...theme.typography.subtitle2, })) const BodyText = styled(Typography)(({ theme }) => ({ color: theme.custom.colors.silverGrayDark, marginBottom: "16px", + ...theme.typography.body2, })) const Footer = styled.div({ diff --git a/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx b/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx index 4a5fae0258..697d3f9ae5 100644 --- a/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx +++ b/frontends/mit-learn/src/pages/SearchPage/SearchPage.test.tsx @@ -306,6 +306,11 @@ describe("SearchPage", () => { setMockResponse.get(urls.userMe.get(), { is_learning_path_editor: true, }) + APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" + APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 + APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 + APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 renderWithProviders() await waitFor(() => { const adminFacetContainer = screen.getByText("Admin Options") @@ -321,6 +326,11 @@ describe("SearchPage", () => { }) test("admin users can set the search mode and slop", async () => { + APP_SETTINGS.DEFAULT_SEARCH_MODE = "phrase" + APP_SETTINGS.DEFAULT_SEARCH_SLOP = 6 + APP_SETTINGS.DEFAULT_SEARCH_STALENESS_PENALTY = 2.5 + APP_SETTINGS.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + APP_SETTINGS.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 90 setMockApiResponses({ search: { count: 700, @@ -349,14 +359,24 @@ test("admin users can set the search mode and slop", async () => { user.click(adminFacetContainer) }) - let slopSlider = screen.queryByText("Slop") - expect(slopSlider).toBeNull() - - const searchModeDropdowns = await screen.findAllByText("best_fields") + const searchModeDropdowns = await screen.findAllByText("phrase") const searchModeDropdown = searchModeDropdowns[0] await user.click(searchModeDropdown) + const mostFieldsSelect = await screen.findByRole("option", { + name: "most_fields", + }) + + await user.click(mostFieldsSelect) + + expect(location.current.search).toBe("?search_mode=most_fields") + + const slopSlider = screen.queryByText("Slop") + expect(slopSlider).toBeNull() + + await user.click(searchModeDropdown) + const phraseSelect = await screen.findByRole("option", { name: "phrase", }) @@ -370,17 +390,6 @@ test("admin users can set the search mode and slop", async () => { }) await user.click(searchModeDropdown) - - const mostFieldsSelect = await screen.findByRole("option", { - name: "most_fields", - }) - - await user.click(mostFieldsSelect) - - expect(location.current.search).toBe("?search_mode=most_fields") - - slopSlider = screen.queryByText("Slop") - expect(slopSlider).toBeNull() }) describe("Search Page Tabs", () => { diff --git a/frontends/mit-learn/webpack.config.js b/frontends/mit-learn/webpack.config.js index 7e49fd3bbb..9d48b30bd0 100644 --- a/frontends/mit-learn/webpack.config.js +++ b/frontends/mit-learn/webpack.config.js @@ -45,6 +45,11 @@ const { CSRF_COOKIE_NAME, APPZI_URL, MITOL_NOINDEX, + DEFAULT_SEARCH_MODE, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, } = cleanEnv(process.env, { NODE_ENV: str({ choices: ["development", "production", "test"], @@ -124,6 +129,26 @@ const { desc: "Whether to include a noindex meta tag", default: true, }), + DEFAULT_SEARCH_SLOP: num({ + desc: "The default search slop", + default: 6, + }), + DEFAULT_SEARCH_STALENESS_PENALTY: num({ + desc: "The default search staleness penalty", + default: 2.5, + }), + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: num({ + desc: "The default search minimum score cutoff", + default: 0, + }), + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: num({ + desc: "The default search max incompleteness penalty", + default: 90, + }), + DEFAULT_SEARCH_MODE: str({ + desc: "The default search mode", + default: "phrase", + }), }) const MITOL_FEATURES_PREFIX = "FEATURE_" @@ -265,6 +290,11 @@ module.exports = (env, argv) => { MITOL_SUPPORT_EMAIL: JSON.stringify(MITOL_SUPPORT_EMAIL), PUBLIC_URL: JSON.stringify(PUBLIC_URL), CSRF_COOKIE_NAME: JSON.stringify(CSRF_COOKIE_NAME), + DEFAULT_SEARCH_MODE: JSON.stringify(DEFAULT_SEARCH_MODE), + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY, + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF, + DEFAULT_SEARCH_SLOP, + DEFAULT_SEARCH_STALENESS_PENALTY, }, }), ] diff --git a/frontends/ol-components/src/components/Button/Button.tsx b/frontends/ol-components/src/components/Button/Button.tsx index ac88f0db5d..641e264f18 100644 --- a/frontends/ol-components/src/components/Button/Button.tsx +++ b/frontends/ol-components/src/components/Button/Button.tsx @@ -12,6 +12,7 @@ type ButtonVariant = | "text" | "noBorder" | "inverted" + | "success" type ButtonSize = "small" | "medium" | "large" type ButtonEdge = "circular" | "rounded" | "none" @@ -127,6 +128,27 @@ const ButtonStyled = styled.button((props) => { borderColor: "currentcolor", borderStyle: "solid", }, + variant === "success" && { + backgroundColor: colors.darkGreen, + color: colors.white, + border: "none", + /* Shadow/04dp */ + boxShadow: + "0px 2px 4px 0px rgba(37, 38, 43, 0.10), 0px 3px 8px 0px rgba(37, 38, 43, 0.12)", + ":hover:not(:disabled)": { + backgroundColor: colors.darkGreen, + boxShadow: "none", + }, + ":disabled": { + backgroundColor: colors.silverGray, + boxShadow: "none", + }, + }, + hasBorder && { + backgroundColor: "transparent", + borderColor: "currentcolor", + borderStyle: "solid", + }, variant === "secondary" && { color: colors.red, ":hover:not(:disabled)": { diff --git a/frontends/ol-components/src/components/Popover/Popover.tsx b/frontends/ol-components/src/components/Popover/Popover.tsx index 48a332c227..39e96ff8af 100644 --- a/frontends/ol-components/src/components/Popover/Popover.tsx +++ b/frontends/ol-components/src/components/Popover/Popover.tsx @@ -128,6 +128,7 @@ const Arrow = styled("div")({ const Content = styled.div(({ theme }) => ({ padding: "16px", backgroundColor: theme.custom.colors.white, + borderRadius: "8px", boxShadow: "0px 2px 4px 0px rgba(37, 38, 43, 0.10), 0px 6px 24px 0px rgba(37, 38, 43, 0.24)", })) diff --git a/frontends/ol-utilities/src/types/settings.d.ts b/frontends/ol-utilities/src/types/settings.d.ts index da51840dd7..553c75e0c8 100644 --- a/frontends/ol-utilities/src/types/settings.d.ts +++ b/frontends/ol-utilities/src/types/settings.d.ts @@ -22,5 +22,10 @@ export declare global { SITE_NAME: string MITOL_SUPPORT_EMAIL: string PUBLIC_URL: string + DEFAULT_SEARCH_MODE: string + DEFAULT_SEARCH_SLOP: number + DEFAULT_SEARCH_STALENESS_PENALTY: number + DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF: number + DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY: number } } diff --git a/learning_resources/management/commands/backpopulate_mit_edx_data.py b/learning_resources/management/commands/backpopulate_mit_edx_data.py index 3886cfe64b..77e183e185 100644 --- a/learning_resources/management/commands/backpopulate_mit_edx_data.py +++ b/learning_resources/management/commands/backpopulate_mit_edx_data.py @@ -22,9 +22,15 @@ def add_arguments(self, parser): help="Delete all existing records first", ) parser.add_argument( - "--api_datafile", - dest="api_datafile", - help="If provided, use this file as the source of API data", + "--api_course_datafile", + dest="api_course_datafile", + help="If provided, use this file as the source of course API data", + default=None, + ) + parser.add_argument( + "--api_program_datafile", + dest="api_program_datafile", + help="If provided, use this file as the source of program API data", default=None, ) super().add_arguments(parser) @@ -40,7 +46,9 @@ def handle(self, *args, **options): # noqa: ARG002 ): resource_delete_actions(learning_resource) else: - task = get_mit_edx_data.delay(options["api_datafile"]) + task = get_mit_edx_data.delay( + options["api_course_datafile"], options["api_program_datafile"] + ) self.stdout.write(f"Started task {task} to get MIT edX course data") self.stdout.write("Waiting on task...") start = now_in_utc() diff --git a/learning_resources/management/commands/extract_openedx_data.py b/learning_resources/management/commands/extract_openedx_data.py index b7f14d3fe1..84cde80f33 100644 --- a/learning_resources/management/commands/extract_openedx_data.py +++ b/learning_resources/management/commands/extract_openedx_data.py @@ -5,13 +5,14 @@ from django.core.management import BaseCommand -from learning_resources.etl import mit_edx, oll +from learning_resources.etl import mit_edx, mit_edx_programs, oll from learning_resources.etl.constants import ETLSource from main.utils import now_in_utc EXTRACTORS = { ETLSource.oll.name: oll.extract, ETLSource.mit_edx.name: mit_edx.extract, + f"{ETLSource.mit_edx.name}_programs": mit_edx_programs.extract, } diff --git a/learning_resources/tasks.py b/learning_resources/tasks.py index 3657875b89..0fd1527a3e 100644 --- a/learning_resources/tasks.py +++ b/learning_resources/tasks.py @@ -48,15 +48,22 @@ def get_micromasters_data(): @app.task -def get_mit_edx_data(api_datafile=None) -> int: +def get_mit_edx_data( + api_course_datafile: str | None = None, api_program_datafile: str | None = None +) -> int: """Task to sync MIT edX data with the database Args: - api_datafile (str): If provided, use this file as the source of API data + api_course_datafile (str): If provided, use file as source of course API data Otherwise, the API is queried directly. + api_program_datafile (str): If provided, use file as source of program API data. + Otherwise, the API is queried directly. + + Returns: + int: The number of results that were fetched """ - courses = pipelines.mit_edx_courses_etl(api_datafile) - programs = pipelines.mit_edx_programs_etl(api_datafile) + courses = pipelines.mit_edx_courses_etl(api_course_datafile) + programs = pipelines.mit_edx_programs_etl(api_program_datafile) clear_search_cache() return len(courses) + len(programs) diff --git a/learning_resources_search/api.py b/learning_resources_search/api.py index eb4582a581..f6e08a07d1 100644 --- a/learning_resources_search/api.py +++ b/learning_resources_search/api.py @@ -5,6 +5,7 @@ from collections import Counter from datetime import UTC, datetime +from django.conf import settings from opensearch_dsl import Search from opensearch_dsl.query import MoreLikeThis, Percolate from opensearchpy.exceptions import NotFoundError @@ -507,7 +508,19 @@ def adjust_original_query_for_percolate(query): Remove keys that are irrelevent when storing original queries for percolate uniqueness such as "limit" and "offset" """ - for key in ["limit", "offset", "sortby", "yearly_decay_percent", "dev_mode"]: + for key in [ + "limit", + "offset", + "sortby", + "yearly_decay_percent", + "dev_mode", + "use_dfs_query_then_fetch", + "max_incompleteness_penalty", + "min_score", + "search_mode", + "slop", + "use_dfs_query_then_fetch", + ]: query.pop(key, None) return order_params(query) @@ -684,6 +697,21 @@ def execute_learn_search(search_params): Returns: dict: The opensearch response dict """ + if search_params.get("endpoint") != CONTENT_FILE_TYPE: + if search_params.get("yearly_decay_percent") is None: + search_params["yearly_decay_percent"] = ( + settings.DEFAULT_SEARCH_STALENESS_PENALTY + ) + if search_params.get("search_mode") is None: + search_params["search_mode"] = settings.DEFAULT_SEARCH_MODE + if search_params.get("slop") is None: + search_params["slop"] = settings.DEFAULT_SEARCH_SLOP + if search_params.get("min_score") is None: + search_params["min_score"] = settings.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF + if search_params.get("max_incompleteness_penalty") is None: + search_params["max_incompleteness_penalty"] = ( + settings.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY + ) search = construct_search(search_params) return search.execute().to_dict() diff --git a/learning_resources_search/api_test.py b/learning_resources_search/api_test.py index 71282429f5..86b810113b 100644 --- a/learning_resources_search/api_test.py +++ b/learning_resources_search/api_test.py @@ -1051,6 +1051,10 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "offset": 1, "sortby": "-readable_id", "endpoint": LEARNING_RESOURCE, + "yearly_decay_percent": 0, + "max_incompleteness_penalty": 0, + "min_score": 0, + "search_mode": "best_fields", } query = { @@ -1079,6 +1083,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1090,6 +1095,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "topics.name" ], + "type": "best_fields", } }, } @@ -1104,6 +1110,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1117,6 +1124,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1132,6 +1140,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -1150,6 +1159,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -1168,6 +1178,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -1194,6 +1205,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1203,6 +1215,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "best_fields", } }, } @@ -1217,6 +1230,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1230,6 +1244,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1245,6 +1260,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -1263,6 +1279,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -1281,6 +1298,7 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -1450,8 +1468,12 @@ def test_execute_learn_search_for_learning_resource_query(opensearch): ], ) def test_execute_learn_search_with_script_score( - mocker, opensearch, yearly_decay_percent, max_incompleteness_penalty + mocker, settings, opensearch, yearly_decay_percent, max_incompleteness_penalty ): + settings.DEFAULT_SEARCH_MODE = "phrase" + settings.DEFAULT_SEARCH_SLOP = 0 + settings.DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = 0 + opensearch.conn.search.return_value = { "hits": {"total": {"value": 10, "relation": "eq"}} } @@ -1531,6 +1553,7 @@ def test_execute_learn_search_with_script_score( "course_feature", "video.transcript.english", ], + "type": "phrase", } }, { @@ -1542,6 +1565,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "topics.name" ], + "type": "phrase", } }, } @@ -1556,6 +1580,7 @@ def test_execute_learn_search_with_script_score( "departments.department_id", "departments.name", ], + "type": "phrase", } }, } @@ -1569,6 +1594,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "course.course_numbers.value" ], + "type": "phrase", } }, } @@ -1584,6 +1610,7 @@ def test_execute_learn_search_with_script_score( "runs.semester", "runs.level", ], + "type": "phrase", } }, } @@ -1602,6 +1629,7 @@ def test_execute_learn_search_with_script_score( "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "phrase", } }, } @@ -1620,6 +1648,7 @@ def test_execute_learn_search_with_script_score( "short_description.english^2", "content_feature_type", ], + "type": "phrase", } }, "score_mode": "avg", @@ -1646,6 +1675,7 @@ def test_execute_learn_search_with_script_score( "course_feature", "video.transcript.english", ], + "type": "phrase", } }, { @@ -1655,6 +1685,7 @@ def test_execute_learn_search_with_script_score( "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "phrase", } }, } @@ -1669,6 +1700,7 @@ def test_execute_learn_search_with_script_score( "departments.department_id", "departments.name", ], + "type": "phrase", } }, } @@ -1682,6 +1714,7 @@ def test_execute_learn_search_with_script_score( "fields": [ "course.course_numbers.value" ], + "type": "phrase", } }, } @@ -1697,6 +1730,7 @@ def test_execute_learn_search_with_script_score( "runs.semester", "runs.level", ], + "type": "phrase", } }, } @@ -1715,6 +1749,7 @@ def test_execute_learn_search_with_script_score( "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "phrase", } }, } @@ -1733,6 +1768,7 @@ def test_execute_learn_search_with_script_score( "short_description.english^2", "content_feature_type", ], + "type": "phrase", } }, "score_mode": "avg", @@ -1898,11 +1934,15 @@ def test_execute_learn_search_with_script_score( ) -def test_execute_learn_search_with_min_score(mocker, opensearch): +def test_execute_learn_search_with_min_score(mocker, settings, opensearch): opensearch.conn.search.return_value = { "hits": {"total": {"value": 10, "relation": "eq"}} } + settings.DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = 0 + settings.DEFAULT_SEARCH_STALENESS_PENALTY = 0 + settings.DEFAULT_SEARCH_MODE = "best_fields" + search_params = { "aggregations": ["offered_by"], "q": "math", @@ -1943,6 +1983,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -1954,6 +1995,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "topics.name" ], + "type": "best_fields", } }, } @@ -1968,6 +2010,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -1981,6 +2024,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -1996,6 +2040,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -2014,6 +2059,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -2032,6 +2078,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", @@ -2058,6 +2105,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "course_feature", "video.transcript.english", ], + "type": "best_fields", } }, { @@ -2067,6 +2115,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "multi_match": { "query": "math", "fields": ["topics.name"], + "type": "best_fields", } }, } @@ -2081,6 +2130,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "departments.department_id", "departments.name", ], + "type": "best_fields", } }, } @@ -2094,6 +2144,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "fields": [ "course.course_numbers.value" ], + "type": "best_fields", } }, } @@ -2109,6 +2160,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.semester", "runs.level", ], + "type": "best_fields", } }, } @@ -2127,6 +2179,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "runs.instructors.last_name^5", "runs.instructors.full_name^5", ], + "type": "best_fields", } }, } @@ -2145,6 +2198,7 @@ def test_execute_learn_search_with_min_score(mocker, opensearch): "short_description.english^2", "content_feature_type", ], + "type": "best_fields", } }, "score_mode": "avg", diff --git a/learning_resources_search/serializers.py b/learning_resources_search/serializers.py index 54225e29fa..fd9bd3fdcf 100644 --- a/learning_resources_search/serializers.py +++ b/learning_resources_search/serializers.py @@ -331,7 +331,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=2.5, help_text=( "Relevance score penalty percent per year for for resources without " "upcoming runs. Only affects results if there is a search term." @@ -425,7 +424,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=0, help_text=( "Minimum score value a text query result needs to have to be displayed" ), @@ -435,7 +433,6 @@ class LearningResourcesSearchRequestSerializer(SearchRequestSerializer): min_value=0, required=False, allow_null=True, - default=0, help_text=( "Maximum score penalty for incomplete OCW courses in percent. " "An OCW course with completeness = 0 will have this score penalty. " diff --git a/main/settings.py b/main/settings.py index e3038ed7a8..c4e3864445 100644 --- a/main/settings.py +++ b/main/settings.py @@ -33,7 +33,7 @@ from main.settings_pluggy import * # noqa: F403 from openapi.settings_spectacular import open_spectacular_settings -VERSION = "0.19.2" +VERSION = "0.19.3" log = logging.getLogger() @@ -766,3 +766,16 @@ def get_all_config_keys(): # Enable or disable search engine indexing MITOL_NOINDEX = get_bool("MITOL_NOINDEX", True) # noqa: FBT003 + +# Search defaults settings - adjustable throught the admin ui +DEFAULT_SEARCH_MODE = get_string(name="DEFAULT_SEARCH_MODE", default="phrase") +DEFAULT_SEARCH_SLOP = get_int(name="DEFAULT_SEARCH_SLOP", default=6) +DEFAULT_SEARCH_STALENESS_PENALTY = get_float( + name="DEFAULT_SEARCH_STALENESS_PENALTY", default=2.5 +) +DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF = get_float( + name="DEFAULT_SEARCH_MINIMUM_SCORE_CUTOFF", default=0 +) +DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY = get_float( + name="DEFAULT_SEARCH_MAX_INCOMPLETENESS_PENALTY", default=90 +) diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index af4d7a9bbf..ea686f47a4 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -2493,7 +2493,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -2506,7 +2505,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -2717,7 +2715,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -2976,7 +2973,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -2989,7 +2985,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -3200,7 +3195,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -3484,7 +3478,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -3497,7 +3490,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -3722,7 +3714,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -3983,7 +3974,6 @@ paths: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness. @@ -3996,7 +3986,6 @@ paths: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed - in: query name: offered_by @@ -4221,7 +4210,6 @@ paths: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. tags: @@ -10055,7 +10043,6 @@ components: maximum: 10 minimum: 0 nullable: true - default: 2.5 description: Relevance score penalty percent per year for for resources without upcoming runs. Only affects results if there is a search term. certification: @@ -10136,7 +10123,6 @@ components: maximum: 50 minimum: 0 nullable: true - default: 0.0 description: Minimum score value a text query result needs to have to be displayed max_incompleteness_penalty: @@ -10145,7 +10131,6 @@ components: maximum: 100 minimum: 0 nullable: true - default: 0.0 description: Maximum score penalty for incomplete OCW courses in percent. An OCW course with completeness = 0 will have this score penalty. Partially complete courses have a linear penalty proportional to the degree of incompleteness.