From 80eb6d6efeeea92df311de9eb531eac6e5d1924a Mon Sep 17 00:00:00 2001 From: Mircea Lungu Date: Mon, 17 Aug 2026 11:39:54 +0300 Subject: [PATCH] Stop filtering discovery by CEFR level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On-demand simplification (#698) quietly turned the feed's level filter into a starvation filter. `available_cefr_levels` lists the levels for which a version of an article was ALREADY generated: the original's own level plus its simplified children. Once crawl-time simplification was retired there are no children, so the field collapsed to a single value and a learner saw only articles a publisher happened to write at exactly their band. Measured on the live index for Danish over 7 days: 33 documents matched an A2 reader, out of 479 in the language. Meanwhile 266 of those articles already carried an A2 ArticleLevelSummary — content written for precisely that reader, which the filter guaranteed they would never see. The premise no longer holds. An article being above the learner's level is not a reason to hide it when the card shows a level-appropriate summary and the body simplifies on request. So drop the filter from all three discovery paths — recommender feed, video recommendations, search — rather than widen it. Above-level readers are the mirror case and get the same treatment once we complexify. Two things this also fixes, both of which produced empty feeds: - Articles whose LLM assessment failed have no cefr_level and so matched nothing. Every assessment outage (the Aug-13 article_type bug, the Anthropic usage-cap outages) therefore emptied feeds outright. Feeds are now independent of assessment liveness. - document_from_video never writes an available_cefr_levels field, so requiring a level in it excluded every video from every leveled user. feed_delivery_health_check reached its per-segment inventory THROUGH the recommender builder, so it would have silently degraded into six copies of one language-wide query. It now appends the CEFR clause itself, and says what that clause means now: a canary on assessment coverage, not on feed delivery. UserPreference.SHOW_EASIER_ARTICLES is inert as a result — nothing left to widen. Left in place, and flagged, until the web toggle is removed. Co-Authored-By: Claude Opus 5 --- tools/analyze_classroom_recommendations.py | 3 - tools/feed_delivery_health_check.py | 50 ++++++++----- .../elastic_recommender.py | 24 +----- zeeguu/core/elastic/elastic_query_builder.py | 75 ++++++++----------- zeeguu/core/model/user_preference.py | 7 +- .../core/test/test_elastic_query_builder.py | 74 ++++++++++++++++++ 6 files changed, 145 insertions(+), 88 deletions(-) create mode 100644 zeeguu/core/test/test_elastic_query_builder.py diff --git a/tools/analyze_classroom_recommendations.py b/tools/analyze_classroom_recommendations.py index 08673cdb0..dbdd7390c 100755 --- a/tools/analyze_classroom_recommendations.py +++ b/tools/analyze_classroom_recommendations.py @@ -112,8 +112,6 @@ def analyze_user_recommendations(user_email, show_details=True, count=20): # Get user constraints for recommendations ( language, - upper_bounds, - lower_bounds, topics_to_include, topics_to_exclude, wanted_searches, @@ -122,7 +120,6 @@ def analyze_user_recommendations(user_email, show_details=True, count=20): ) = _prepare_user_constraints(user) print(f" Language: {language.name if language else 'Not set'}") - print(f" Difficulty Level: {lower_bounds/10:.1f} - {upper_bounds/10:.1f}") # Topics print(f"\n Subscribed Topics:") diff --git a/tools/feed_delivery_health_check.py b/tools/feed_delivery_health_check.py index 658373aeb..7030575a2 100644 --- a/tools/feed_delivery_health_check.py +++ b/tools/feed_delivery_health_check.py @@ -23,18 +23,19 @@ Two signals per segment ------------------------ 1. INVENTORY (primary alert trigger): re-run the app's OWN recommender query - builder (build_elastic_recommender_query) parameterized by the segment's - language + CEFR level, with NO per-user seen-exclusion and NO topic - narrowing. This is "does fresh content that this segment COULD be shown even - exist in the index?" It replicates the exact CEFR filtering - (available_cefr_levels) that starved the incident's segment, without being - confounded by one user's reading history. + builder (build_elastic_recommender_query) for the segment's language, with NO + per-user seen-exclusion and NO topic narrowing, plus a CEFR clause this check + adds itself. This is "does fresh content that this segment COULD be shown even + exist in the index?", without being confounded by one user's reading history. + The CEFR half of it has since become a check on ASSESSMENT rather than on + delivery — the feed no longer filters by level (see + segment_inventory_published_times). 2. PER-USER CANARY (corroboration): pick a real, recently-active user in the segment and call the REAL feed function article_recommendations_for_user(). - This exercises the full path the app uses — CEFR filtering, the feature flags - (simplified-only vs originals-first, "show easier"), the disturbing filter, - the ES index, seen-exclusion, teacher-upload filtering. + This exercises the full path the app uses — the feature flags (simplified-only + vs originals-first), the disturbing filter, the ES index, seen-exclusion, + teacher-upload filtering. INVENTORY is the primary trigger because a single power-user can legitimately have an empty PERSONAL feed (they already opened everything fresh) — that's a @@ -269,13 +270,20 @@ def active_users_by_segment(active_days, language_codes=None): def segment_inventory_published_times(segment, count): """Fresh-inventory probe for a segment, independent of any one user. - Reuses the app's OWN query builder (build_elastic_recommender_query) with the - segment's language + CEFR level and NO per-user filters (no seen-exclusion, - no topic narrowing, no disturbing filter). include_lower=False on purpose: - the incident hit users who see ONLY their exact level, so the strict same- - level query is the worst-case signal we want to alarm on. + Reuses the app's OWN query builder (build_elastic_recommender_query) for the + segment's language with NO per-user filters (no seen-exclusion, no topic + narrowing, no disturbing filter), then adds the CEFR clause on top. - LIMITATION: this models CEFR filtering but NOT per-user feature-flag filtering + That clause is ours now, not the feed's: the recommender stopped filtering on + available_cefr_levels once simplification became on-demand (any article is + readable at any level), so nothing in the delivery path narrows by level + anymore. We keep probing per level because the field still tells us whether + CEFR ASSESSMENT is running — a segment going empty here means new articles + are landing unassessed (the Aug 2026 article_type bug, and the Anthropic + usage-cap outages, both looked exactly like this), which no longer starves + the feed but does starve level summaries and the level badge. + + LIMITATION: this models CEFR coverage but NOT per-user feature-flag filtering (e.g. a simplified-only feed). A flag-driven emptiness — like the ORIGINAL Aug 2026 incident (simplified-only feed, no simplified content) — would show only in the per-user canary, not here. That's acceptable because api#698 made @@ -288,6 +296,7 @@ def segment_inventory_published_times(segment, count): from elasticsearch import Elasticsearch from zeeguu.core.elastic.elastic_query_builder import ( build_elastic_recommender_query, + get_cefr_levels_to_match, ) from zeeguu.core.elastic.settings import ES_CONN_STRING, ES_ZINDEX @@ -298,7 +307,6 @@ def segment_inventory_published_times(segment, count): user_topics="", unwanted_user_topics="", language=language, - user_cefr_level=segment.cefr_level, es_scale="1d", es_offset="1d", es_decay=0.6, @@ -307,9 +315,17 @@ def segment_inventory_published_times(segment, count): user_ignored_sources=[], articles_to_exclude=None, filter_disturbing=False, - include_lower=False, page=0, ) + query_body["query"]["function_score"]["query"]["bool"]["must"].append( + { + "terms": { + "available_cefr_levels.keyword": get_cefr_levels_to_match( + segment.cefr_level + ) + } + } + ) es = Elasticsearch(ES_CONN_STRING) res = es.search(index=ES_ZINDEX, body=query_body) diff --git a/zeeguu/core/content_recommender/elastic_recommender.py b/zeeguu/core/content_recommender/elastic_recommender.py index 0e7eabb0c..0fd80d47b 100644 --- a/zeeguu/core/content_recommender/elastic_recommender.py +++ b/zeeguu/core/content_recommender/elastic_recommender.py @@ -46,9 +46,6 @@ def _prepare_user_constraints(user, language=None): if language is None: language = user.learned_language - # Get user's CEFR level for filtering (e.g., "A1", "B2") - user_cefr_level = user.cefr_level_for_language(language) - # 1. Unwanted user topics # ============================== user_search_filters = SearchFilter.all_for_user(user) @@ -90,7 +87,6 @@ def _prepare_user_constraints(user, language=None): return ( language, - user_cefr_level, _topics_to_string(topics_to_include), _topics_to_string(topics_to_exclude), _list_to_string(wanted_user_searches), @@ -138,7 +134,6 @@ def article_recommendations_for_user( final_article_mix = [] ( language, - user_cefr_level, topics_to_include, topics_to_exclude, wanted_user_searches, @@ -158,17 +153,12 @@ def article_recommendations_for_user( # Check if user has enabled disturbing content filtering filter_disturbing = UserPreference.is_filter_disturbing_content_enabled(user) - # "Show easier articles": also surface articles below the user's level (helps - # advanced/native readers and thin-inventory languages avoid an empty feed). - include_lower = UserPreference.is_show_easier_articles_enabled(user) - # build the query using elastic_query_builder query_body = build_elastic_recommender_query( count, wanted_user_searches, unwanted_user_searches, language, - user_cefr_level, es_scale, es_offset, es_decay, @@ -177,7 +167,6 @@ def article_recommendations_for_user( user_ignored_sources=user_ignored_sources, articles_to_exclude=articles_to_exclude, filter_disturbing=filter_disturbing, - include_lower=include_lower, page=page, ) @@ -279,7 +268,6 @@ def video_recommendations_for_user( ): ( language, - user_cefr_level, topics_to_include, topics_to_exclude, wanted_user_searches, @@ -293,7 +281,6 @@ def video_recommendations_for_user( wanted_user_searches, unwanted_user_searches, language, - user_cefr_level, topics_to_include=topics_to_include, topics_to_exclude=topics_to_exclude, user_ignored_sources=user_ignored_sources, @@ -322,7 +309,6 @@ def article_and_video_search_for_user( ( language, - user_cefr_level, topics_to_include, topics_to_exclude, wanted_user_searches, @@ -335,7 +321,6 @@ def article_and_video_search_for_user( count, search_terms, language, - user_cefr_level, es_time_scale, es_time_offset, es_time_decay, @@ -376,8 +361,6 @@ def topic_filter_for_user( difficulty_level, topic, ): - from zeeguu.core.elastic.elastic_query_builder import get_cefr_levels_to_match - es = Elasticsearch(ES_CONN_STRING) s = Search().query(Q("term", language=user.learned_language.code())) @@ -406,12 +389,7 @@ def topic_filter_for_user( if topic != None and topic != "all": s = s.filter("match", topics=topic.lower()) - # Filter by user's CEFR level; the "show easier articles" pref also includes - # everything below the user's level. - user_cefr_level = user.cefr_level_for_learned_language() - include_lower = UserPreference.is_show_easier_articles_enabled(user) - levels_to_match = get_cefr_levels_to_match(user_cefr_level, include_lower) - s = s.filter("terms", **{"available_cefr_levels.keyword": levels_to_match}) + # No CEFR filter here either — see build_elastic_recommender_query for why. query = s.query diff --git a/zeeguu/core/elastic/elastic_query_builder.py b/zeeguu/core/elastic/elastic_query_builder.py index 26afe4bfd..e08cb02e9 100644 --- a/zeeguu/core/elastic/elastic_query_builder.py +++ b/zeeguu/core/elastic/elastic_query_builder.py @@ -50,26 +50,21 @@ def more_like_this_query(count, article_text, language, page=0): CEFR_LEVEL_ORDER = ["A1", "A2", "B1", "B2", "C1", "C2"] -def get_cefr_levels_to_match(user_cefr_level, include_lower=False): +def get_cefr_levels_to_match(user_cefr_level): """ Returns the list of CEFR levels that match the user's level. Includes exact match and compound levels where user is the upper half. E.g., A2 user matches: ["A2", "A1/A2"]. - When include_lower is True, ALSO match every level BELOW the user's — so an - advanced/native reader (or a thin-inventory language) isn't starved down to - only same-level articles. E.g., C1 user matches every band from A1 up to - "B2/C1". Driven by the "show easier articles" user preference. + NOTE: discovery (feed, topic browsing, search) no longer filters on CEFR at + all — see build_elastic_recommender_query. What is left of this function + serves tools/feed_delivery_health_check.py, which still probes per-level + inventory as a canary on CEFR assessment coverage. """ - idx = CEFR_LEVEL_ORDER.index(user_cefr_level) - exact_levels = CEFR_LEVEL_ORDER[: idx + 1] if include_lower else [user_cefr_level] - - levels = [] - for lvl in exact_levels: - levels.append(lvl) - i = CEFR_LEVEL_ORDER.index(lvl) - if i > 0: - levels.append(f"{CEFR_LEVEL_ORDER[i - 1]}/{lvl}") + levels = [user_cefr_level] + i = CEFR_LEVEL_ORDER.index(user_cefr_level) + if i > 0: + levels.append(f"{CEFR_LEVEL_ORDER[i - 1]}/{user_cefr_level}") return levels @@ -78,7 +73,6 @@ def build_elastic_recommender_query( user_topics, unwanted_user_topics, language, - user_cefr_level, es_scale, es_offset, es_decay, @@ -87,7 +81,6 @@ def build_elastic_recommender_query( user_ignored_sources, articles_to_exclude=None, filter_disturbing=False, - include_lower=False, page=0, ): """ @@ -95,16 +88,23 @@ def build_elastic_recommender_query( Filters articles by: - Language - - User's CEFR level (via available_cefr_levels field) - Topic preferences - Disturbing content (if enabled) Scores/ranks by recency (preferring recent articles). - Args: - user_cefr_level: User's CEFR level string (e.g., "A1", "B2") - Articles must have this level in available_cefr_levels. - Also matches compound levels (e.g., A2 matches "A1/A2"). + NOT filtered by CEFR level, deliberately. With on-demand simplification + every article is readable at the learner's level — the feed card carries a + level-appropriate summary (ArticleLevelSummary) and the body simplifies on + request (POST /simplify_article/) — so an article being "too hard" is no + longer a reason to hide it. Filtering on available_cefr_levels used to mean + the opposite: that field lists the levels for which a version was already + generated, so once crawl-time simplification was retired it collapsed to the + original's own level and a learner saw only articles a publisher happened to + write at exactly their band (for Danish A2: ~3/day out of ~50). It also hid + every article whose LLM assessment failed, which turned any assessment + outage into an empty feed. Above-level readers are the mirror case and get + the same treatment once we complexify. """ # must = mandatory, has to occur @@ -166,13 +166,6 @@ def build_elastic_recommender_query( if filter_disturbing: must_not.append({"match": {"is_disturbing": True}}) - # Filter by CEFR level. Normally only articles available at the user's level; - # with include_lower (the "show easier articles" pref) also everything below. - # Use .keyword sub-field for exact matching (field is mapped as text) - if user_cefr_level: - levels_to_match = get_cefr_levels_to_match(user_cefr_level, include_lower) - must.append(terms("available_cefr_levels.keyword", levels_to_match)) - must.append(exists("published_time")) # Allow both articles and videos in organic recommendations must.append({"bool": {"should": [exists("article_id"), exists("video_id")]}}) @@ -219,15 +212,19 @@ def build_elastic_search_query_for_videos( user_topics, unwanted_user_topics, language, - user_cefr_level, topics_to_include, topics_to_exclude, user_ignored_sources, page, ): """ - Builds video search query with CEFR level filtering. - Similar to article recommender but with less emphasis on recency. + Builds video search query. Similar to article recommender but with less + emphasis on recency. + + No CEFR filter — same reasoning as build_elastic_recommender_query, and for + videos the filter was worse than redundant: document_from_video never writes + an available_cefr_levels field, so requiring a level in it excluded every + video from every leveled user's results. """ must = [] @@ -262,12 +259,6 @@ def build_elastic_search_query_for_videos( ) ) - # Filter by CEFR level - only show videos available at user's level - # Use .keyword sub-field for exact matching (field is mapped as text) - if user_cefr_level: - levels_to_match = get_cefr_levels_to_match(user_cefr_level) - must.append(terms("available_cefr_levels.keyword", levels_to_match)) - must.append(exists("published_time")) must.append(exists("video_id")) @@ -309,7 +300,6 @@ def build_elastic_search_query( count, search_terms, language, - user_cefr_level=None, es_time_scale="1d", es_time_offset="1d", es_time_decay=0.65, @@ -319,7 +309,9 @@ def build_elastic_search_query( """ Builds an elastic search query for search terms. - Filters by CEFR level and ranks by recency. + Ranks by recency. No CEFR filter — a learner searching for a word wants the + articles that contain it, and any of them can be simplified on demand (same + reasoning as build_elastic_recommender_query). """ s = ( @@ -334,11 +326,6 @@ def build_elastic_search_query( .exclude("match", description="pg15") ) - # Add CEFR level filter (use .keyword sub-field for exact matching) - if user_cefr_level: - levels_to_match = get_cefr_levels_to_match(user_cefr_level) - s = s.filter("terms", **{"available_cefr_levels.keyword": levels_to_match}) - # using function scores to weight more recent results higher # https://github.com/elastic/elasticsearch-dsl-py/issues/608 preferences = [] diff --git a/zeeguu/core/model/user_preference.py b/zeeguu/core/model/user_preference.py index 57d68b943..6c133425c 100644 --- a/zeeguu/core/model/user_preference.py +++ b/zeeguu/core/model/user_preference.py @@ -148,7 +148,12 @@ def is_email_on_article_shared_enabled(cls, user: User): def is_show_easier_articles_enabled(cls, user: User): """Whether to ALSO show articles below the user's CEFR level (not just at level). Helps advanced/native readers and thin-inventory languages whose - at-level feed would otherwise be near-empty. Default: False.""" + at-level feed would otherwise be near-empty. Default: False. + + INERT since the feed stopped filtering by CEFR level altogether (see + build_elastic_recommender_query) — every article is already shown, so + there is nothing left for this to widen. Kept until the web toggle in + Settings > Feed Preferences is removed or repurposed.""" setting = UserPreference.query.filter_by( user_id=user.id, key=cls.SHOW_EASIER_ARTICLES ).first() diff --git a/zeeguu/core/test/test_elastic_query_builder.py b/zeeguu/core/test/test_elastic_query_builder.py new file mode 100644 index 000000000..093df8bb8 --- /dev/null +++ b/zeeguu/core/test/test_elastic_query_builder.py @@ -0,0 +1,74 @@ +""" +The recommender/search queries deliberately do NOT narrow by CEFR level. + +With on-demand simplification any article is readable at the learner's level, so +filtering on `available_cefr_levels` — which lists only the levels ALREADY +generated for an article — hid nearly the whole index from a learner (and hid +everything at all whenever LLM assessment was down). These tests pin that down so +the filter can't creep back in unnoticed; see build_elastic_recommender_query. +""" + +import json + +from zeeguu.core.elastic.elastic_query_builder import ( + build_elastic_recommender_query, + build_elastic_search_query, + build_elastic_search_query_for_videos, + get_cefr_levels_to_match, +) + + +class FakeLanguage: + name = "Danish" + + +def recommender_query(): + return build_elastic_recommender_query( + 20, + user_topics="", + unwanted_user_topics="", + language=FakeLanguage(), + es_scale="1d", + es_offset="1d", + es_decay=0.6, + topics_to_include="", + topics_to_exclude="", + user_ignored_sources=[], + ) + + +def test_recommender_query_does_not_filter_by_cefr_level(): + assert "available_cefr_levels" not in json.dumps(recommender_query()) + + +def test_recommender_query_still_filters_by_language(): + must = recommender_query()["query"]["function_score"]["query"]["bool"]["must"] + assert {"match": {"language": "Danish"}} in must + + +def test_search_query_does_not_filter_by_cefr_level(): + query = build_elastic_search_query(20, "hygge", FakeLanguage()) + assert "available_cefr_levels" not in json.dumps(query) + + +def test_video_query_does_not_filter_by_cefr_level(): + # Videos are the sharpest case: document_from_video never writes an + # available_cefr_levels field, so a terms filter on it matched no video ever. + query = build_elastic_search_query_for_videos( + 20, + user_topics="", + unwanted_user_topics="", + language=FakeLanguage(), + topics_to_include="", + topics_to_exclude="", + user_ignored_sources=[], + page=0, + ) + assert "available_cefr_levels" not in json.dumps(query) + + +def test_cefr_levels_to_match_includes_the_compound_band_below(): + # Still used by tools/feed_delivery_health_check.py to probe per-level + # assessment coverage. + assert get_cefr_levels_to_match("A2") == ["A2", "A1/A2"] + assert get_cefr_levels_to_match("A1") == ["A1"]