diff --git a/channels/views_test.py b/channels/views_test.py index 705312fcdd..ef2d8b08cb 100644 --- a/channels/views_test.py +++ b/channels/views_test.py @@ -30,6 +30,7 @@ User = get_user_model() +@pytest.mark.skip_nplusone_check def test_list_channels(user_client): """Test that all channels are returned""" ChannelFactory.create_batch(2, published=False) # should be filtered out @@ -119,6 +120,7 @@ def test_partial_update_channel_featured_list_only_learning_path( assert response.status_code == status +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("resource_type", LearningResourceType) def test_create_channel_lists_only_learning_path(admin_client, resource_type): """Only learning_paths may be used as one of lists""" @@ -449,6 +451,7 @@ def test_channel_configuration_is_not_editable(client, channel): assert channel.configuration == initial_config +@pytest.mark.skip_nplusone_check def test_channel_counts_view(client): """Test the channel counts view returns counts for resources""" url = reverse( @@ -486,6 +489,7 @@ def test_channel_counts_view(client): ) +@pytest.mark.skip_nplusone_check def test_channel_counts_view_is_cached_for_anonymous_users(client, settings): """Test the channel counts view is cached for anonymous users""" settings.CACHES["redis"] = { @@ -508,6 +512,7 @@ def test_channel_counts_view_is_cached_for_anonymous_users(client, settings): assert len(response) == channel_count +@pytest.mark.skip_nplusone_check def test_channel_counts_view_is_cached_for_authenticated_users(client, settings): """Test the channel counts view is cached for authenticated users""" settings.CACHES["redis"] = { diff --git a/fixtures/common.py b/fixtures/common.py index 8b86e3d7d6..8abcb83ae8 100644 --- a/fixtures/common.py +++ b/fixtures/common.py @@ -10,6 +10,7 @@ import responses from pytest_mock import PytestMockWarning from urllib3.exceptions import InsecureRequestWarning +from zeal import zeal_ignore from channels.factories import ChannelUnitDetailFactory from learning_resources.constants import LearningResourceRelationTypes, OfferedBy @@ -122,3 +123,13 @@ def offeror_featured_lists(): channel = ChannelUnitDetailFactory.create(unit=offeror).channel channel.featured_list = featured_path channel.save() + + +@pytest.fixture(autouse=True) +def check_nplusone(request): + """Raise nplusone errors""" + if request.node.get_closest_marker("skip_nplusone_check"): + with zeal_ignore(): + yield + else: + yield diff --git a/learning_resources/filters_test.py b/learning_resources/filters_test.py index 4c86131e37..a496d899a6 100644 --- a/learning_resources/filters_test.py +++ b/learning_resources/filters_test.py @@ -534,6 +534,7 @@ def test_learning_resource_filter_delivery(mock_courses, client): ) +@pytest.mark.skip_nplusone_check def test_content_file_filter_run_id(mock_content_files, client): """Test that the run_id filter works for contentfiles""" @@ -553,6 +554,7 @@ def test_content_file_filter_run_id(mock_content_files, client): ) +@pytest.mark.skip_nplusone_check def test_content_file_filter_resource_id(mock_content_files, client): """Test that the resource_id filter works for contentfiles""" @@ -575,6 +577,7 @@ def test_content_file_filter_resource_id(mock_content_files, client): ) +@pytest.mark.skip_nplusone_check def test_content_file_filter_edx_module_id(mock_content_files, client): """Test that the resource_id filter works for contentfiles""" assert mock_content_files[0].edx_module_id is None @@ -597,6 +600,7 @@ def test_content_file_filter_edx_module_id(mock_content_files, client): ] +@pytest.mark.skip_nplusone_check def test_content_file_filter_platform(mock_content_files, client): """Test that the platform filter works""" @@ -614,6 +618,7 @@ def test_content_file_filter_platform(mock_content_files, client): ) +@pytest.mark.skip_nplusone_check def test_content_file_filter_offered_by(mock_content_files, client): """Test that the offered_by filter works for contentfiles""" @@ -631,6 +636,7 @@ def test_content_file_filter_offered_by(mock_content_files, client): ) +@pytest.mark.skip_nplusone_check def test_learning_resource_filter_content_feature_type(client): """Test that the resource_content_tag filter works""" diff --git a/learning_resources/views_learningpath_test.py b/learning_resources/views_learningpath_test.py index f7336d2f74..ca574c3a1c 100644 --- a/learning_resources/views_learningpath_test.py +++ b/learning_resources/views_learningpath_test.py @@ -32,6 +32,7 @@ def mock_opensearch(mocker): ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("is_public", [True, False]) @pytest.mark.parametrize("is_editor", [True, False]) @pytest.mark.parametrize("has_image", [True, False]) @@ -135,6 +136,7 @@ def test_learning_path_endpoint_create( # pylint: disable=too-many-arguments # assert resp.data.get("description") == resp.data.get("description") +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("is_public", [True, False]) @pytest.mark.parametrize("is_editor", [True, False]) @pytest.mark.parametrize("update_topics", [True, False]) @@ -175,6 +177,7 @@ def test_learning_path_endpoint_patch(client, update_topics, is_public, is_edito ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("is_editor", [True, False]) def test_learning_path_items_endpoint_create_item(client, user, is_editor): """Test lr_learningpathitems_api endpoint for creating a LearningPath item""" @@ -232,6 +235,7 @@ def test_learning_path_items_endpoint_create_item_bad_data(client, user): } +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( ("is_editor", "position"), [[True, 0], [True, 2], [False, 1]], # noqa: PT007 @@ -301,6 +305,7 @@ def test_learning_path_items_endpoint_update_items_wrong_list(client, user): assert resp.status_code == 404 +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("num_items", [2, 3]) @pytest.mark.parametrize("is_editor", [True, False]) def test_learning_path_items_endpoint_delete_items(client, user, is_editor, num_items): @@ -334,6 +339,7 @@ def test_learning_path_items_endpoint_delete_items(client, user, is_editor, num_ assert item.position == (old_position - 1 if is_editor else old_position) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("is_editor", [True, False]) def test_learning_path_endpoint_delete(client, user, is_editor): """Test learningpath endpoint for deleting a LearningPath""" @@ -414,6 +420,7 @@ def test_get_resource_learning_paths(user_client, user, is_editor): assert response_data == expected +@pytest.mark.skip_nplusone_check def test_set_learning_path_relationships(client, staff_user): """Test the learning_paths endpoint for setting multiple userlist relationships""" course = factories.CourseFactory.create() @@ -440,6 +447,7 @@ def test_set_learning_path_relationships(client, staff_user): ).exists() +@pytest.mark.skip_nplusone_check def test_adding_to_learning_path_not_effect_existing_membership(client, staff_user): """ Given L1 (existing parent), L2 (new parent), and R (resource), diff --git a/learning_resources/views_test.py b/learning_resources/views_test.py index fc9cee2586..9d2f858a6c 100644 --- a/learning_resources/views_test.py +++ b/learning_resources/views_test.py @@ -113,6 +113,7 @@ def test_get_course_detail_endpoint(client, url): ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( "url", [ @@ -139,6 +140,7 @@ def test_get_course_content_files_endpoint(client, url): assert resp.data.get("results")[idx]["id"] == content_file.id +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( ("reverse_url", "expected_url"), [ @@ -264,6 +266,7 @@ def test_no_excess_offeror_queries(client, django_assert_num_queries, offeror_co assert result["channel_url"] is not None +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( "user_role", [ @@ -311,6 +314,7 @@ def test_list_content_files_list_endpoint(client, user_role, django_user_model): assert result.get("content") is None +@pytest.mark.skip_nplusone_check def test_list_content_files_list_endpoint_with_no_runs(client): """Test ContentFile list endpoint returns results even without associated runs""" course = CourseFactory.create() @@ -330,6 +334,7 @@ def test_list_content_files_list_endpoint_with_no_runs(client): assert result["id"] in content_file_ids +@pytest.mark.skip_nplusone_check def test_list_content_files_list_filtered(client): """Test ContentFile list endpoint""" course_1 = CourseFactory.create() @@ -505,6 +510,7 @@ def test_get_podcast_episode_detail_endpoint(client, url): ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( "url", ["lr:v1:learning_resource_items_api-list", "lr:v1:podcast_items_api-list"] ) @@ -836,6 +842,7 @@ def test_offerors_detail_endpoint(client): assert resp.data == LearningResourceOfferorDetailSerializer(instance=offeror).data +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( ("url", "params"), [ @@ -879,6 +886,7 @@ def test_get_video_playlist_detail_endpoint(client, url): ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( "url", ["lr:v1:learning_resource_items_api-list", "lr:v1:video_playlist_items_api-list"], @@ -923,6 +931,7 @@ def test_get_video_playlist_items_endpoint(client, url): ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( ("url", "params"), [ @@ -948,6 +957,7 @@ def test_list_video_endpoint(client, url, params): ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( ("url", "params"), [ @@ -1228,6 +1238,7 @@ def test_similar_resources_endpoint_ignores_opensearch_published(mocker, client) assert len(response_ids) == 4 +@pytest.mark.skip_nplusone_check def test_vector_similar_resources_endpoint_does_not_return_self(mocker, client): """Test vector based similar resources endpoint does not return initial id""" from learning_resources.models import LearningResource @@ -1298,6 +1309,7 @@ def test_vector_similar_resources_endpoint_only_returns_published(mocker, client assert len(response_ids) == 1 +@pytest.mark.skip_nplusone_check def test_learning_resources_display_info_list_view(mocker, client): """Test learning_resources_display_info_list_view returns expected results""" from learning_resources.models import LearningResource diff --git a/learning_resources/views_userlist_test.py b/learning_resources/views_userlist_test.py index 4102745f1b..dbb25b12d5 100644 --- a/learning_resources/views_userlist_test.py +++ b/learning_resources/views_userlist_test.py @@ -136,6 +136,7 @@ def test_user_list_endpoint_membership_get(client, user, is_authenticated): assert resp.status_code == 403 +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("is_author", [True, False]) def test_user_list_items_endpoint_create_item(client, user, is_author): """Test userlistitems endpoint for creating a UserListItem""" @@ -177,6 +178,7 @@ def test_user_list_items_endpoint_create_item_bad_data(client, user): } +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( ("is_author", "position"), [[True, 0], [True, 2], [False, 1]], # noqa: PT007 @@ -309,6 +311,7 @@ def test_get_resource_user_lists(client, user, is_author, is_unlisted): assert items_json == [] +@pytest.mark.skip_nplusone_check def test_set_userlist_relationships(client, user): """Test the userlists endpoint for setting multiple userlist relationships""" course = factories.CourseFactory.create() @@ -376,6 +379,7 @@ def assign_userlists(course, userlists): ) +@pytest.mark.skip_nplusone_check def test_adding_to_userlist_not_effect_existing_membership(client, user): """ Given L1 (existing parent), L2 (new parent), and R (resource), diff --git a/main/settings.py b/main/settings.py index e30284f27e..229296bec8 100644 --- a/main/settings.py +++ b/main/settings.py @@ -200,6 +200,17 @@ "django_scim.middleware.SCIMAuthCheckMiddleware", ) +ZEAL_ENABLE = get_bool("ZEAL_ENABLE", False) # noqa: FBT003 + +# enable the zeal nplusone profiler only in debug mode or under pytest +if ZEAL_ENABLE or ENVIRONMENT == "pytest": + INSTALLED_APPS += ("zeal",) + # this should be the first middleware so we catch any issues in our own middleware + MIDDLEWARE += ("zeal.middleware.zeal_middleware",) + +ZEAL_RAISE = False +ZEAL_ALLOWLIST = [] + # CORS CORS_ALLOWED_ORIGINS = get_list_of_str("CORS_ALLOWED_ORIGINS", []) CORS_ALLOWED_ORIGIN_REGEXES = get_list_of_str("CORS_ALLOWED_ORIGIN_REGEXES", []) @@ -483,6 +494,7 @@ "opensearch": {"level": OS_LOG_LEVEL}, "nplusone": {"handlers": ["console"], "level": "ERROR"}, "boto3": {"handlers": ["console"], "level": "ERROR"}, + "zeal": {"handlers": ["console"], "level": "ERROR"}, }, "root": {"handlers": ["console"], "level": LOG_LEVEL}, } diff --git a/news_events/filters_test.py b/news_events/filters_test.py index 07792ccbce..4aac79b4f0 100644 --- a/news_events/filters_test.py +++ b/news_events/filters_test.py @@ -9,6 +9,7 @@ ITEM_API_URL = "/api/v0/news_events/" +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize( "multifilter", ["feed_type={}&feed_type={}", "feed_type={},{}"] ) diff --git a/news_events/views_test.py b/news_events/views_test.py index 0e6396dff4..2f08dfe5fa 100644 --- a/news_events/views_test.py +++ b/news_events/views_test.py @@ -11,6 +11,7 @@ from news_events.factories import FeedEventDetailFactory +@pytest.mark.skip_nplusone_check def test_feed_source_viewset_list(client): """Est that the feed sources list viewset returns data in expected format""" sources = sorted(factories.FeedSourceFactory.create_batch(5), key=lambda x: x.id) @@ -25,6 +26,7 @@ def test_feed_source_viewset_list(client): ) +@pytest.mark.skip_nplusone_check @pytest.mark.parametrize("feed_type", FeedType.names()) def test_feed_source_viewset_list_filtered(client, feed_type): """Test that the sources list viewset returns data filtered by feed type""" diff --git a/poetry.lock b/poetry.lock index 3c6e89f988..567a88279e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1784,6 +1784,18 @@ files = [ {file = "django_webpack_loader-3.1.1-py2.py3-none-any.whl", hash = "sha256:15c05cb685b113c5e6f947efa8ee3888a03c2969a2dd0da4c7610b3e695f67ba"}, ] +[[package]] +name = "django-zeal" +version = "2.0.4" +description = "Detect N+1s in your Django app" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "django_zeal-2.0.4-py3-none-any.whl", hash = "sha256:c37b0a7d203572d8eb8462a601f19dedfdf82324a25c7eb7273afdc52fc519fa"}, + {file = "django_zeal-2.0.4.tar.gz", hash = "sha256:934cd3968c1ede3f3ec5314c4da285ef4796f4b6de1f95870c5999370280e17e"}, +] + [[package]] name = "djangorestframework" version = "3.16.0" @@ -9243,4 +9255,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.1" python-versions = "~3.12" -content-hash = "e12fa65a8bb0e0fce970adf4f007880d4c9d61be763510a5e477be41385288d7" +content-hash = "477c3027b6a958243d04bc6ab0ff3269b2c2a423534c4ec33d4dd3cba39f518f" diff --git a/profiles/views_test.py b/profiles/views_test.py index 2a8f8ef426..d4c80c4e9e 100644 --- a/profiles/views_test.py +++ b/profiles/views_test.py @@ -237,6 +237,7 @@ def test_patch_profile_by_user(client, logged_in_profile): assert logged_in_profile.location == location_json +@pytest.mark.skip_nplusone_check def test_patch_topic_interests(client, logged_in_profile): """Test that patching Profile.topic_interests works correctly""" topics = LearningResourceTopicFactory.create_batch(3) diff --git a/pyproject.toml b/pyproject.toml index b9b81e760c..77391d6546 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -114,6 +114,7 @@ wrapt = "^1.14.1" youtube-transcript-api = "^1.0.0" pypdfium2 = "^4.30.0" pyarrow = "^21.0.0" +django-zeal = "^2.0.4" diff --git a/pytest.ini b/pytest.ini index 44c42e250e..cb5c692192 100644 --- a/pytest.ini +++ b/pytest.ini @@ -12,6 +12,7 @@ env = OPENSEARCH_INDEX=testindex MITOL_COOKIE_DOMAIN=localhost MITOL_COOKIE_NAME=cookie_monster + MITOL_ENVIRONMENT=pytest MITOL_FEATURES_DEFAULT=False MITOL_SECURE_SSL_REDIRECT=False MITOL_USE_S3=False diff --git a/uwsgi.ini b/uwsgi.ini index 29991bdc80..823e716296 100644 --- a/uwsgi.ini +++ b/uwsgi.ini @@ -27,6 +27,6 @@ socket-timeout = 3 endif = buffer-size = 65535 stats = /tmp/uwsgi-stats.sock -reload-on-rss = 300 +reload-on-rss = 400 # for sentry py-call-uwsgi-fork-hooks = true