From f9e7f2f9a3bfe22e45eea3594ab4a9a180a18863 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Wed, 11 Sep 2024 10:30:53 -0400 Subject: [PATCH 01/11] adding view cache to learning resource view --- learning_resources/views.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/learning_resources/views.py b/learning_resources/views.py index adabf4954d..ccf9b053b3 100644 --- a/learning_resources/views.py +++ b/learning_resources/views.py @@ -91,7 +91,7 @@ AnonymousAccessReadonlyPermission, is_admin_user, ) -from main.utils import chunks +from main.utils import cache_page_for_anonymous_users, chunks log = logging.getLogger(__name__) @@ -159,6 +159,14 @@ def get_queryset(self) -> QuerySet: """ return self._get_base_queryset().filter(published=True) + @method_decorator( + cache_page_for_anonymous_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def dispatch(self, request, *args, **kwargs): + return super().dispatch(request, *args, **kwargs) + @extend_schema_view( list=extend_schema( From c04908430257127d783b28ffdc542bb6e95c0c9c Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Wed, 11 Sep 2024 15:41:30 -0400 Subject: [PATCH 02/11] caching more views --- articles/views.py | 11 +++++ channels/views.py | 4 +- learning_resources/views.py | 83 +++++++++++++++++++++++++++++++++++-- 3 files changed, 93 insertions(+), 5 deletions(-) diff --git a/articles/views.py b/articles/views.py index e5c88c0601..39423c152d 100644 --- a/articles/views.py +++ b/articles/views.py @@ -1,3 +1,5 @@ +from django.conf import settings +from django.utils.decorators import method_decorator from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import viewsets from rest_framework.pagination import LimitOffsetPagination @@ -6,6 +8,7 @@ from articles.models import Article from articles.serializers import ArticleSerializer from main.constants import VALID_HTTP_METHODS +from main.utils import cache_page_for_all_users # Create your views here. @@ -37,3 +40,11 @@ class ArticleViewSet(viewsets.ModelViewSet): permission_classes = [IsAdminUser] http_method_names = VALID_HTTP_METHODS + + @method_decorator( + cache_page_for_all_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) diff --git a/channels/views.py b/channels/views.py index 2fc2114211..d730b380f8 100644 --- a/channels/views.py +++ b/channels/views.py @@ -28,7 +28,7 @@ from learning_resources.views import DefaultPagination from main.constants import VALID_HTTP_METHODS from main.permissions import AnonymousAccessReadonlyPermission -from main.utils import cache_page_for_all_users +from main.utils import cache_page_for_anonymous_users log = logging.getLogger(__name__) @@ -218,7 +218,7 @@ def get_queryset(self): ) @method_decorator( - cache_page_for_all_users( + cache_page_for_anonymous_users( settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" ) ) diff --git a/learning_resources/views.py b/learning_resources/views.py index ccf9b053b3..766dbf0871 100644 --- a/learning_resources/views.py +++ b/learning_resources/views.py @@ -91,7 +91,7 @@ AnonymousAccessReadonlyPermission, is_admin_user, ) -from main.utils import cache_page_for_anonymous_users, chunks +from main.utils import cache_page_for_all_users, cache_page_for_anonymous_users, chunks log = logging.getLogger(__name__) @@ -164,8 +164,8 @@ def get_queryset(self) -> QuerySet: settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" ) ) - def dispatch(self, request, *args, **kwargs): - return super().dispatch(request, *args, **kwargs) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) @extend_schema_view( @@ -386,6 +386,14 @@ class ResourceListItemsViewSet(NestedViewSetMixin, viewsets.ReadOnlyModelViewSet filter_backends = [OrderingFilter] ordering = ["position", "-child__last_modified"] + @method_decorator( + cache_page_for_anonymous_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + @extend_schema( parameters=[ @@ -516,6 +524,14 @@ def learning_paths(self, request, *args, **kwargs): # noqa: ARG002 serializer = SerializerClass(current_relationships, many=True) return Response(serializer.data) + @method_decorator( + cache_page_for_anonymous_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + @extend_schema_view( create=extend_schema(summary="Learning Path Resource Relationship Add"), @@ -541,6 +557,14 @@ class LearningPathItemsViewSet(ResourceListItemsViewSet, viewsets.ModelViewSet): permission_classes = (permissions.HasLearningPathItemPermissions,) http_method_names = VALID_HTTP_METHODS + @method_decorator( + cache_page_for_anonymous_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + def create(self, request, *args, **kwargs): request.data["parent"] = request.data.get("parent_id") return super().create(request, *args, **kwargs) @@ -576,6 +600,14 @@ class TopicViewSet(viewsets.ReadOnlyModelViewSet): filter_backends = [DjangoFilterBackend] filterset_class = TopicFilter + @method_decorator( + cache_page_for_anonymous_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + @extend_schema_view( list=extend_schema(summary="List"), @@ -827,6 +859,14 @@ class ContentTagViewSet(viewsets.ReadOnlyModelViewSet): pagination_class = LargePagination permission_classes = (AnonymousAccessReadonlyPermission,) + @method_decorator( + cache_page_for_all_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, *args, **kwargs): + return super().list(*args, **kwargs) + @extend_schema_view( list=extend_schema(summary="List"), @@ -846,6 +886,14 @@ class DepartmentViewSet(viewsets.ReadOnlyModelViewSet): lookup_url_kwarg = "department_id" lookup_field = "department_id__iexact" + @method_decorator( + cache_page_for_all_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, *args, **kwargs): + return super().list(*args, **kwargs) + @extend_schema_view( list=extend_schema(summary="List"), @@ -861,6 +909,14 @@ class SchoolViewSet(viewsets.ReadOnlyModelViewSet): pagination_class = LargePagination permission_classes = (AnonymousAccessReadonlyPermission,) + @method_decorator( + cache_page_for_all_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, *args, **kwargs): + return super().list(*args, **kwargs) + @extend_schema_view( list=extend_schema(summary="List"), @@ -876,6 +932,14 @@ class PlatformViewSet(viewsets.ReadOnlyModelViewSet): pagination_class = LargePagination permission_classes = (AnonymousAccessReadonlyPermission,) + @method_decorator( + cache_page_for_all_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, *args, **kwargs): + return super().list(*args, **kwargs) + @extend_schema_view( list=extend_schema(summary="List"), @@ -892,6 +956,14 @@ class OfferedByViewSet(viewsets.ReadOnlyModelViewSet): permission_classes = (AnonymousAccessReadonlyPermission,) lookup_field = "code" + @method_decorator( + cache_page_for_anonymous_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + @extend_schema_view( list=extend_schema( @@ -1002,6 +1074,11 @@ def get_queryset(self) -> QuerySet: .distinct() ) + @method_decorator( + cache_page_for_anonymous_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) @extend_schema( summary="List", description="Get a paginated list of featured resources", From 4504b8023c398f4326b778beed85a9dd948f926d Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 12 Sep 2024 11:31:11 -0400 Subject: [PATCH 03/11] updating spec --- frontends/api/src/generated/v1/api.ts | 12 ++++-------- openapi/specs/v1.yaml | 3 +-- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/frontends/api/src/generated/v1/api.ts b/frontends/api/src/generated/v1/api.ts index 4ce15dede9..21e09ba257 100644 --- a/frontends/api/src/generated/v1/api.ts +++ b/frontends/api/src/generated/v1/api.ts @@ -16195,8 +16195,7 @@ export const LearningpathsApiAxiosParamCreator = function ( } }, /** - * Get a list of related learning resources for a learning resource. - * @summary Nested Learning Resource List + * Viewset for LearningPath related resources * @param {number} learning_resource_id The learning resource id of the learning path * @param {number} [limit] Number of results to return per page. * @param {number} [offset] The initial index from which to return the results. @@ -16779,8 +16778,7 @@ export const LearningpathsApiFp = function (configuration?: Configuration) { )(axios, operationBasePath || basePath) }, /** - * Get a list of related learning resources for a learning resource. - * @summary Nested Learning Resource List + * Viewset for LearningPath related resources * @param {number} learning_resource_id The learning resource id of the learning path * @param {number} [limit] Number of results to return per page. * @param {number} [offset] The initial index from which to return the results. @@ -17133,8 +17131,7 @@ export const LearningpathsApiFactory = function ( .then((request) => request(axios, basePath)) }, /** - * Get a list of related learning resources for a learning resource. - * @summary Nested Learning Resource List + * Viewset for LearningPath related resources * @param {LearningpathsApiLearningpathsItemsListRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. * @throws {RequiredError} @@ -17673,8 +17670,7 @@ export class LearningpathsApi extends BaseAPI { } /** - * Get a list of related learning resources for a learning resource. - * @summary Nested Learning Resource List + * Viewset for LearningPath related resources * @param {LearningpathsApiLearningpathsItemsListRequest} requestParameters Request parameters. * @param {*} [options] Override http request option. * @throws {RequiredError} diff --git a/openapi/specs/v1.yaml b/openapi/specs/v1.yaml index 8e9711f615..4e6b173689 100644 --- a/openapi/specs/v1.yaml +++ b/openapi/specs/v1.yaml @@ -4923,8 +4923,7 @@ paths: /api/v1/learningpaths/{learning_resource_id}/items/: get: operationId: learningpaths_items_list - description: Get a list of related learning resources for a learning resource. - summary: Nested Learning Resource List + description: Viewset for LearningPath related resources parameters: - in: path name: learning_resource_id From 06f97c3b14ecce8b31d356df27cbd52e3d4baee6 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 12 Sep 2024 14:06:49 -0400 Subject: [PATCH 04/11] switching to cache page for all users --- channels/views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channels/views.py b/channels/views.py index d730b380f8..2fc2114211 100644 --- a/channels/views.py +++ b/channels/views.py @@ -28,7 +28,7 @@ from learning_resources.views import DefaultPagination from main.constants import VALID_HTTP_METHODS from main.permissions import AnonymousAccessReadonlyPermission -from main.utils import cache_page_for_anonymous_users +from main.utils import cache_page_for_all_users log = logging.getLogger(__name__) @@ -218,7 +218,7 @@ def get_queryset(self): ) @method_decorator( - cache_page_for_anonymous_users( + cache_page_for_all_users( settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" ) ) From df77e8e7c12096266b6d455c894a13d8671bbff2 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Thu, 12 Sep 2024 14:56:06 -0400 Subject: [PATCH 05/11] fixing tests --- channels/views_test.py | 15 +++++++++++++-- conftest.py | 9 +++++++++ learning_resources/views.py | 8 -------- main/utils.py | 9 +++++---- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/channels/views_test.py b/channels/views_test.py index 12b4e1d7dc..00c0334e7a 100644 --- a/channels/views_test.py +++ b/channels/views_test.py @@ -483,8 +483,14 @@ def test_channel_counts_view(client): ) -def test_channel_counts_view_is_cached_for_anonymous_users(client): +def test_channel_counts_view_is_cached_for_anonymous_users(client, settings): """Test the channel counts view is cached for anonymous users""" + settings.CACHES["redis"] = { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": settings.CELERY_BROKER_URL, + "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, + } + channel_count = 5 channels = ChannelFactory.create_batch(channel_count, channel_type="unit") url = reverse( @@ -499,8 +505,13 @@ def test_channel_counts_view_is_cached_for_anonymous_users(client): assert len(response) == channel_count -def test_channel_counts_view_is_cached_for_authenticated_users(client): +def test_channel_counts_view_is_cached_for_authenticated_users(client, settings): """Test the channel counts view is cached for authenticated users""" + settings.CACHES["redis"] = { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": settings.CELERY_BROKER_URL, + "OPTIONS": {"CLIENT_CLASS": "django_redis.client.DefaultClient"}, + } channel_count = 5 channel_user = UserFactory.create() client.force_login(channel_user) diff --git a/conftest.py b/conftest.py index 0ea2a33f4d..c569bd3fce 100644 --- a/conftest.py +++ b/conftest.py @@ -20,3 +20,12 @@ def prevent_requests(mocker, request): # noqa: PT004 autospec=True, side_effect=DoNotUseRequestException, ) + + +@pytest.fixture(autouse=True) +def _use_dummy_redis_cache_backend(settings): + new_cache_settings = settings.CACHES.copy() + new_cache_settings["redis"] = { + "BACKEND": "django.core.cache.backends.dummy.DummyCache", + } + settings.CACHES = new_cache_settings diff --git a/learning_resources/views.py b/learning_resources/views.py index 766dbf0871..48bd344c44 100644 --- a/learning_resources/views.py +++ b/learning_resources/views.py @@ -386,14 +386,6 @@ class ResourceListItemsViewSet(NestedViewSetMixin, viewsets.ReadOnlyModelViewSet filter_backends = [OrderingFilter] ordering = ["position", "-child__last_modified"] - @method_decorator( - cache_page_for_anonymous_users( - settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" - ) - ) - def list(self, request, *args, **kwargs): - return super().list(request, *args, **kwargs) - @extend_schema( parameters=[ diff --git a/main/utils.py b/main/utils.py index 5c4cf8216d..d227c63f93 100644 --- a/main/utils.py +++ b/main/utils.py @@ -357,7 +357,8 @@ def clean_data(data: str) -> str: def clear_search_cache(): cache = caches["redis"] - search_keys = cache.keys("views.decorators.cache.cache_page.search.*") - cache.delete_many(search_keys) - search_keys = cache.keys("views.decorators.cache.cache_header.search.*") - cache.delete_many(search_keys) + if hasattr(cache, "keys"): + search_keys = cache.keys("views.decorators.cache.cache_page.search.*") + cache.delete_many(search_keys) + search_keys = cache.keys("views.decorators.cache.cache_header.search.*") + cache.delete_many(search_keys) From e6a3ca607217123dfaf77a65f369b7eda0219867 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Fri, 13 Sep 2024 13:50:00 -0400 Subject: [PATCH 06/11] switching to cache page for all --- learning_resources/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/learning_resources/views.py b/learning_resources/views.py index 48bd344c44..aa1fdad051 100644 --- a/learning_resources/views.py +++ b/learning_resources/views.py @@ -593,7 +593,7 @@ class TopicViewSet(viewsets.ReadOnlyModelViewSet): filterset_class = TopicFilter @method_decorator( - cache_page_for_anonymous_users( + cache_page_for_all_users( settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" ) ) From 79776fa012e73ff06fe593e6bdbb76a25e450713 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Fri, 13 Sep 2024 15:35:12 -0400 Subject: [PATCH 07/11] adding cache clearing to management commands --- .../management/commands/populate_featured_lists.py | 3 ++- .../management/commands/update_departments_schools.py | 3 ++- learning_resources/management/commands/update_offered_by.py | 3 ++- learning_resources/management/commands/update_platforms.py | 3 ++- news_events/management/commands/backpopulate_news_events.py | 2 ++ 5 files changed, 10 insertions(+), 4 deletions(-) diff --git a/learning_resources/management/commands/populate_featured_lists.py b/learning_resources/management/commands/populate_featured_lists.py index a209b0b2dd..4f33a9b157 100644 --- a/learning_resources/management/commands/populate_featured_lists.py +++ b/learning_resources/management/commands/populate_featured_lists.py @@ -16,7 +16,7 @@ LearningResource, LearningResourceOfferor, ) -from main.utils import now_in_utc +from main.utils import clear_cache, now_in_utc class Command(BaseCommand): @@ -90,3 +90,4 @@ def handle(self, *args, **options): # noqa: ARG002 "Population of unit channel featured lists finished, " f"took {total_seconds} seconds" ) + clear_cache() diff --git a/learning_resources/management/commands/update_departments_schools.py b/learning_resources/management/commands/update_departments_schools.py index 6d6de62542..8832959531 100644 --- a/learning_resources/management/commands/update_departments_schools.py +++ b/learning_resources/management/commands/update_departments_schools.py @@ -6,7 +6,7 @@ upsert_department_data, upsert_school_data, ) -from main.utils import now_in_utc +from main.utils import clear_cache, now_in_utc class Command(BaseCommand): @@ -26,3 +26,4 @@ def handle(self, *args, **options): # noqa: ARG002 f"Update of {len(schools)} schools & {len(departments)} " f"departments finished, took {total_seconds} seconds" ) + clear_cache() diff --git a/learning_resources/management/commands/update_offered_by.py b/learning_resources/management/commands/update_offered_by.py index 0088c1476e..826ed1b129 100644 --- a/learning_resources/management/commands/update_offered_by.py +++ b/learning_resources/management/commands/update_offered_by.py @@ -3,7 +3,7 @@ from django.core.management import BaseCommand from learning_resources.utils import upsert_offered_by_data -from main.utils import now_in_utc +from main.utils import clear_cache, now_in_utc class Command(BaseCommand): @@ -21,3 +21,4 @@ def handle(self, *args, **options): # noqa: ARG002 self.stdout.write( f"Update of {len(offerors)} offerors finished, took {total_seconds} seconds" ) + clear_cache() diff --git a/learning_resources/management/commands/update_platforms.py b/learning_resources/management/commands/update_platforms.py index d1af4774e4..309071a180 100644 --- a/learning_resources/management/commands/update_platforms.py +++ b/learning_resources/management/commands/update_platforms.py @@ -3,7 +3,7 @@ from django.core.management import BaseCommand from learning_resources.utils import upsert_platform_data -from main.utils import now_in_utc +from main.utils import clear_cache, now_in_utc class Command(BaseCommand): @@ -21,3 +21,4 @@ def handle(self, *args, **options): # noqa: ARG002 self.stdout.write( f"Upserted {len(platform_codes)} platforms, took {total_seconds} seconds" ) + clear_cache() diff --git a/news_events/management/commands/backpopulate_news_events.py b/news_events/management/commands/backpopulate_news_events.py index 7002cea57d..595efd0b37 100644 --- a/news_events/management/commands/backpopulate_news_events.py +++ b/news_events/management/commands/backpopulate_news_events.py @@ -4,6 +4,7 @@ from django.core.management import BaseCommand +from main.utils import clear_cache from news_events.etl import pipelines from news_events.models import FeedImage, FeedSource @@ -55,3 +56,4 @@ def handle(self, *args, **options): # noqa: ARG002 f"Processed {etl_func} pipeline with {item_count} items" ) self.stdout.write("Finished running news/events ETL pipelines") + clear_cache() From f2f87626bc50960a184c12aebbfae9aa6824b144 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Mon, 16 Sep 2024 09:47:14 -0400 Subject: [PATCH 08/11] clear search cache when new articles are created --- articles/views.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/articles/views.py b/articles/views.py index 39423c152d..472fa9f3a1 100644 --- a/articles/views.py +++ b/articles/views.py @@ -8,7 +8,7 @@ from articles.models import Article from articles.serializers import ArticleSerializer from main.constants import VALID_HTTP_METHODS -from main.utils import cache_page_for_all_users +from main.utils import cache_page_for_all_users, clear_search_cache # Create your views here. @@ -48,3 +48,11 @@ class ArticleViewSet(viewsets.ModelViewSet): ) def list(self, request, *args, **kwargs): return super().list(request, *args, **kwargs) + + def create(self, request, *args, **kwargs): + clear_search_cache() + return super().create(request, *args, **kwargs) + + def destroy(self, request, *args, **kwargs): + clear_search_cache() + return super().destroy(request, *args, **kwargs) From 6107960c6a66cdea3a25869fecc63b3dea6e9263 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Mon, 16 Sep 2024 09:54:27 -0400 Subject: [PATCH 09/11] adding caching to uncached views --- news_events/views.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/news_events/views.py b/news_events/views.py index 3277c7f044..ebe7882519 100644 --- a/news_events/views.py +++ b/news_events/views.py @@ -1,11 +1,14 @@ """Views for news_events""" +from django.conf import settings +from django.utils.decorators import method_decorator from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import viewsets from rest_framework.pagination import LimitOffsetPagination from main.filters import MultipleOptionsFilterBackend from main.permissions import AnonymousAccessReadonlyPermission +from main.utils import cache_page_for_all_users from news_events.filters import FeedItemFilter, FeedSourceFilter from news_events.models import FeedItem, FeedSource from news_events.serializers import FeedItemSerializer, FeedSourceSerializer @@ -49,6 +52,14 @@ class FeedItemViewSet(viewsets.ReadOnlyModelViewSet): ) ) + @method_decorator( + cache_page_for_all_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) + @extend_schema_view( list=extend_schema( @@ -70,3 +81,11 @@ class FeedSourceViewSet(viewsets.ReadOnlyModelViewSet): filter_backends = [MultipleOptionsFilterBackend] filterset_class = FeedSourceFilter queryset = FeedSource.objects.all().order_by("id") + + @method_decorator( + cache_page_for_all_users( + settings.SEARCH_PAGE_CACHE_DURATION, cache="redis", key_prefix="search" + ) + ) + def list(self, request, *args, **kwargs): + return super().list(request, *args, **kwargs) From b9619048acf7a4e8dfc4acb716d9bf1858738044 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Mon, 16 Sep 2024 10:03:11 -0400 Subject: [PATCH 10/11] clearing cache after news etl pipelines --- news_events/tasks.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/news_events/tasks.py b/news_events/tasks.py index 935d37bb91..c685e2861c 100644 --- a/news_events/tasks.py +++ b/news_events/tasks.py @@ -1,6 +1,7 @@ """Tasks for news_events""" from main.celery import app +from main.utils import clear_cache from news_events.etl import pipelines @@ -8,33 +9,39 @@ def get_medium_mit_news(): """Run the Medium MIT News ETL pipeline""" pipelines.medium_mit_news_etl() + clear_cache() @app.task def get_ol_events(): """Run the Open Learning Events ETL pipeline""" pipelines.ol_events_etl() + clear_cache() @app.task def get_sloan_exec_news(): """Run the Sloan executive education news ETL pipeline""" pipelines.sloan_exec_news_etl() + clear_cache() @app.task def get_sloan_exec_webinars(): """Run the Sloan webinars ETL pipeline""" pipelines.sloan_webinars_etl() + clear_cache() @app.task def get_mitpe_news(): """Run the MIT Professional Education news ETL pipeline""" pipelines.mitpe_news_etl() + clear_cache() @app.task def get_mitpe_events(): """Run the MIT Professional Education events ETL pipeline""" pipelines.mitpe_events_etl() + clear_cache() From da13a7e22e1684086a7a667a0046103791d15689 Mon Sep 17 00:00:00 2001 From: shankar ambady Date: Mon, 16 Sep 2024 10:14:07 -0400 Subject: [PATCH 11/11] fix method name --- .../commands/backpopulate_news_events.py | 4 ++-- news_events/tasks.py | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/news_events/management/commands/backpopulate_news_events.py b/news_events/management/commands/backpopulate_news_events.py index 595efd0b37..193539ee21 100644 --- a/news_events/management/commands/backpopulate_news_events.py +++ b/news_events/management/commands/backpopulate_news_events.py @@ -4,7 +4,7 @@ from django.core.management import BaseCommand -from main.utils import clear_cache +from main.utils import clear_search_cache from news_events.etl import pipelines from news_events.models import FeedImage, FeedSource @@ -56,4 +56,4 @@ def handle(self, *args, **options): # noqa: ARG002 f"Processed {etl_func} pipeline with {item_count} items" ) self.stdout.write("Finished running news/events ETL pipelines") - clear_cache() + clear_search_cache() diff --git a/news_events/tasks.py b/news_events/tasks.py index c685e2861c..932927ffc7 100644 --- a/news_events/tasks.py +++ b/news_events/tasks.py @@ -1,7 +1,7 @@ """Tasks for news_events""" from main.celery import app -from main.utils import clear_cache +from main.utils import clear_search_cache from news_events.etl import pipelines @@ -9,39 +9,39 @@ def get_medium_mit_news(): """Run the Medium MIT News ETL pipeline""" pipelines.medium_mit_news_etl() - clear_cache() + clear_search_cache() @app.task def get_ol_events(): """Run the Open Learning Events ETL pipeline""" pipelines.ol_events_etl() - clear_cache() + clear_search_cache() @app.task def get_sloan_exec_news(): """Run the Sloan executive education news ETL pipeline""" pipelines.sloan_exec_news_etl() - clear_cache() + clear_search_cache() @app.task def get_sloan_exec_webinars(): """Run the Sloan webinars ETL pipeline""" pipelines.sloan_webinars_etl() - clear_cache() + clear_search_cache() @app.task def get_mitpe_news(): """Run the MIT Professional Education news ETL pipeline""" pipelines.mitpe_news_etl() - clear_cache() + clear_search_cache() @app.task def get_mitpe_events(): """Run the MIT Professional Education events ETL pipeline""" pipelines.mitpe_events_etl() - clear_cache() + clear_search_cache()