From 3cde9f962762b788450e52030d8da6e0afb0637a Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 13 Nov 2025 15:59:10 +0200 Subject: [PATCH 1/4] Fixed duplicated geo levels when importing non covidcast indicator sets --- src/base/resources.py | 47 ++++++++++++++++++---- src/fixtures/geographic_granularities.json | 23 ++++++++--- src/indicatorsets/admin.py | 26 ++++++++++++ src/indicatorsets/resources.py | 10 +++-- 4 files changed, 88 insertions(+), 18 deletions(-) diff --git a/src/base/resources.py b/src/base/resources.py index 1db049c..5ea2930 100644 --- a/src/base/resources.py +++ b/src/base/resources.py @@ -9,6 +9,11 @@ "display_order_number": 2, "short_name": "HHS", }, + "hhs-region": { + "display_name": "U.S. HHS Region", + "display_order_number": 2, + "short_name": "HHS", + }, "census-region": { "display_name": "U.S. Census Region", "display_order_number": 3, @@ -34,40 +39,66 @@ "display_order_number": 7, "short_name": "ADM 3", }, - "hrr": { + "hsa": { "display_name": "Hospital Referral Region (HRR)", "display_order_number": 8, + "short_name": "HSA-NCI", + }, + "hrr": { + "display_name": "Hospital Referral Region (HRR)", + "display_order_number": 9, "short_name": "HRR", }, "msa": { "display_name": "Metropolitan Statistical Area (MSA)", - "display_order_number": 9, + "display_order_number": 10, "short_name": "MSA", }, "dma": { "display_name": "Designated Market Area (DMA)", - "display_order_number": 10, + "display_order_number": 11, "short_name": "DMA", }, "other_substate_region": { "display_name": "Other Substate Region", - "display_order_number": 11, + "display_order_number": 12, "short_name": "Other Substate Region", }, "FluSurv-Net site": { "display_name": "FluSurv-Net site (see documentation)", - "display_order_number": 12, + "display_order_number": 13, "short_name": "FluSurv-Net site", }, "facility": { "display_name": 'Hospital ("Facility")', - "display_order_number": 13, + "display_order_number": 14, "short_name": "Facility", }, "lat_long": { "display_name": "Lat/Long", - "display_order_number": 14, + "display_order_number": 15, "short_name": "Lat/Long", }, - "N/A": {"display_name": "N/A", "display_order_number": 15, "short_name": "N/A"}, + "N/A": {"display_name": "N/A", "display_order_number": 16, "short_name": "N/A"}, } + + +def get_geographic_mapping_by_name(name): + """ + Get geographic granularity mapping by key or by display_name. + + First tries to get by key, then searches by display_name if not found. + Returns a tuple (key, value) where key is the dictionary key and value + is the mapping dict, or None if not found. + """ + # First try to get by key + mapping = GEOGRAPHIC_GRANULARITY_MAPPING.get(name) + if mapping: + return (name, mapping) + + # If not found, search by display_name + for key, value in GEOGRAPHIC_GRANULARITY_MAPPING.items(): + if value.get("display_name") == name: + return (key, value) + + return None diff --git a/src/fixtures/geographic_granularities.json b/src/fixtures/geographic_granularities.json index 9e2cd51..6d43dd0 100644 --- a/src/fixtures/geographic_granularities.json +++ b/src/fixtures/geographic_granularities.json @@ -76,13 +76,24 @@ "short_name": "ADM 3" } }, + { + "model": "base.Geography", + "pk": 14, + "fields": { + "name": "hsa", + "display_name": "Health Service Area (HSA-NCI)", + "display_order_number": 8, + "used_in": "indicatorsets", + "short_name": "HSA-NCI" + } + }, { "model": "base.Geography", "pk": 8, "fields": { "name": "hrr", "display_name": "Hospital Referral Region (HRR)", - "display_order_number": 8, + "display_order_number": 9, "used_in": "indicatorsets", "short_name": "HRR" } @@ -93,7 +104,7 @@ "fields": { "name": "msa", "display_name": "Metropolitan Statistical Area (MSA)", - "display_order_number": 9, + "display_order_number": 10, "used_in": "indicatorsets", "short_name": "MSA" } @@ -104,7 +115,7 @@ "fields": { "name": "dma", "display_name": "Designated Market Area (DMA)", - "display_order_number": 10, + "display_order_number": 11, "used_in": "indicatorsets", "short_name": "DMA" } @@ -115,7 +126,7 @@ "fields": { "name": "facility", "display_name": "Hospital (\"Facility\")", - "display_order_number": 11, + "display_order_number": 12, "used_in": "indicatorsets", "short_name": "Facility" } @@ -126,7 +137,7 @@ "fields": { "name": "FluSurv-Net site", "display_name": "FluSurv-Net site (see documentation)", - "display_order_number": 12, + "display_order_number": 13, "used_in": "indicatorsets", "short_name": "FluSurv-Net site" } @@ -137,7 +148,7 @@ "fields": { "name": "N/A", "display_name": "N/A", - "display_order_number": 13, + "display_order_number": 14, "used_in": "indicatorsets", "short_name": "N/A" } diff --git a/src/indicatorsets/admin.py b/src/indicatorsets/admin.py index d4996e0..08ba6a2 100644 --- a/src/indicatorsets/admin.py +++ b/src/indicatorsets/admin.py @@ -3,6 +3,7 @@ from django.urls import path from import_export.admin import ImportExportModelAdmin +from base.models import Geography, Pathogen, SeverityPyramidRung from base.utils import download_source_file, import_data from indicatorsets.models import ( ColumnDescription, @@ -45,6 +46,31 @@ def get_queryset(self, request): qs = super().get_queryset(request) return qs.exclude(source_type="non_delphi") + def formfield_for_manytomany(self, db_field, request, **kwargs): + """ + Filter geographic_levels field to show only a subset of Geography objects. + Modify the filter criteria as needed. + """ + if db_field.name == "geographic_levels": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = Geography.objects.filter( + used_in="indicatorsets" + ).order_by("display_order_number") + if db_field.name == "pathogens": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = Pathogen.objects.filter( + used_in="indicatorsets" + ).order_by("display_order_number") + if db_field.name == "severity_pyramid_rungs": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = SeverityPyramidRung.objects.filter( + used_in="indicatorsets" + ).order_by("display_order_number") + return super().formfield_for_manytomany(db_field, request, **kwargs) + change_list_template = "admin/indicatorsets/indicator_set_changelist.html" def get_urls(self): diff --git a/src/indicatorsets/resources.py b/src/indicatorsets/resources.py index e09f870..86fc934 100644 --- a/src/indicatorsets/resources.py +++ b/src/indicatorsets/resources.py @@ -4,7 +4,7 @@ from import_export.widgets import ForeignKeyWidget, ManyToManyWidget from base.models import GeographicScope, Geography, Pathogen, SeverityPyramidRung -from base.resources import GEOGRAPHIC_GRANULARITY_MAPPING +from base.resources import get_geographic_mapping_by_name from indicatorsets.models import ( IndicatorSet, NonDelphiIndicatorSet, @@ -70,9 +70,11 @@ def process_available_geographies(row) -> None: for geography in available_geographies: geography_name = geography.strip() default_params = {"used_in": "indicatorsets"} - try: - default_params.update(GEOGRAPHIC_GRANULARITY_MAPPING[geography_name]) - except KeyError: + result = get_geographic_mapping_by_name(geography_name) + if result: + geography_name, mapping = result + default_params.update(mapping) + else: max_display_order_number = Geography.objects.filter( used_in="indicatorsets" ).aggregate(Max("display_order_number"))["display_order_number__max"] From 5e8f3801d2f1b66a45a4a2e13a3f278392e59c06 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 13 Nov 2025 16:08:24 +0200 Subject: [PATCH 2/4] Added objects filtring for indicators admin page --- src/indicators/admin.py | 68 +++++++++++++++++++++++++++++--------- src/indicatorsets/admin.py | 58 ++++++++++++++++---------------- 2 files changed, 82 insertions(+), 44 deletions(-) diff --git a/src/indicators/admin.py b/src/indicators/admin.py index 39bf66a..1f8abf6 100644 --- a/src/indicators/admin.py +++ b/src/indicators/admin.py @@ -3,15 +3,26 @@ from django.urls import path from import_export.admin import ImportExportModelAdmin +from base.models import Geography, Pathogen, SeverityPyramidRung from base.utils import download_source_file, import_data -from indicators.models import (Category, FormatType, Indicator, - IndicatorGeography, IndicatorType, - NonDelphiIndicator, OtherEndpointIndicator, - USStateIndicator) -from indicators.resources import (IndicatorBaseResource, IndicatorResource, - NonDelphiIndicatorResource, - OtherEndpointIndicatorResource, - USStateIndicatorResource) +from indicators.models import ( + Category, + FormatType, + Indicator, + IndicatorGeography, + IndicatorType, + NonDelphiIndicator, + OtherEndpointIndicator, + USStateIndicator, +) +from indicators.resources import ( + IndicatorBaseResource, + IndicatorResource, + NonDelphiIndicatorResource, + OtherEndpointIndicatorResource, + USStateIndicatorResource, +) + @admin.register(IndicatorType) class IndicatorTypeAdmin(admin.ModelAdmin): @@ -47,8 +58,35 @@ class IndicatorGeographyAdmin(admin.ModelAdmin): list_select_related = True +class BaseIndicatorAdmin(ImportExportModelAdmin): + def formfield_for_manytomany(self, db_field, request, **kwargs): + """ + Filter geographic_levels field to show only a subset of Geography objects. + Modify the filter criteria as needed. + """ + if db_field.name == "geographic_levels": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = Geography.objects.filter( + used_in="indicators" + ).order_by("display_order_number") + if db_field.name == "pathogens": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = Pathogen.objects.filter(used_in="indicators").order_by( + "display_order_number" + ) + if db_field.name == "severity_pyramid_rungs": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = SeverityPyramidRung.objects.filter( + used_in="indicators" + ).order_by("display_order_number") + return super().formfield_for_manytomany(db_field, request, **kwargs) + + @admin.register(Indicator) -class IndicatorAdmin(ImportExportModelAdmin): +class IndicatorAdmin(BaseIndicatorAdmin): list_display = ( "name", "description", @@ -102,7 +140,7 @@ def download_indicator(self, request): @admin.register(OtherEndpointIndicator) -class OtherEndpointIndicatorAdmin(ImportExportModelAdmin): +class OtherEndpointIndicatorAdmin(BaseIndicatorAdmin): list_display = ( "name", "description", @@ -138,9 +176,7 @@ def get_urls(self): ), path( "download-source-file", - self.admin_site.admin_view( - self.download_other_endpoint_indicator - ), + self.admin_site.admin_view(self.download_other_endpoint_indicator), name="download_other_endpoint_indicator", ), ] @@ -162,7 +198,7 @@ def download_other_endpoint_indicator(self, request): @admin.register(NonDelphiIndicator) -class NonDelphiIndicatorAdmin(ImportExportModelAdmin): +class NonDelphiIndicatorAdmin(BaseIndicatorAdmin): list_display = ( "name", "member_name", @@ -214,7 +250,7 @@ def download_nondelphi_indicator(self, request): @admin.register(USStateIndicator) -class USStateIndicatorAdmin(ImportExportModelAdmin): +class USStateIndicatorAdmin(BaseIndicatorAdmin): list_display = ("name", "indicator_set") search_fields = ("name", "indicator_set") ordering = ("name",) @@ -257,4 +293,4 @@ def download_us_state_indicator(self, request): return download_source_file( settings.SPREADSHEET_URLS["us_state_indicators"], "US_State_Indicators.csv", - ) \ No newline at end of file + ) diff --git a/src/indicatorsets/admin.py b/src/indicatorsets/admin.py index 08ba6a2..85d8183 100644 --- a/src/indicatorsets/admin.py +++ b/src/indicatorsets/admin.py @@ -16,9 +16,36 @@ from indicatorsets.resources import USStateIndicatorSetResource +class BaseIndicatorSetAdmin(ImportExportModelAdmin): + def formfield_for_manytomany(self, db_field, request, **kwargs): + """ + Filter geographic_levels field to show only a subset of Geography objects. + Modify the filter criteria as needed. + """ + if db_field.name == "geographic_levels": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = Geography.objects.filter( + used_in="indicatorsets" + ).order_by("display_order_number") + if db_field.name == "pathogens": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = Pathogen.objects.filter( + used_in="indicatorsets" + ).order_by("display_order_number") + if db_field.name == "severity_pyramid_rungs": + # Filter to show only geographies used in indicatorsets + # You can modify this filter to show a different subset + kwargs["queryset"] = SeverityPyramidRung.objects.filter( + used_in="indicatorsets" + ).order_by("display_order_number") + return super().formfield_for_manytomany(db_field, request, **kwargs) + + # Register your models here. @admin.register(IndicatorSet) -class IndicatorSetAdmin(ImportExportModelAdmin): +class IndicatorSetAdmin(BaseIndicatorSetAdmin): """ Admin interface for the IndicatorSet model. """ @@ -46,31 +73,6 @@ def get_queryset(self, request): qs = super().get_queryset(request) return qs.exclude(source_type="non_delphi") - def formfield_for_manytomany(self, db_field, request, **kwargs): - """ - Filter geographic_levels field to show only a subset of Geography objects. - Modify the filter criteria as needed. - """ - if db_field.name == "geographic_levels": - # Filter to show only geographies used in indicatorsets - # You can modify this filter to show a different subset - kwargs["queryset"] = Geography.objects.filter( - used_in="indicatorsets" - ).order_by("display_order_number") - if db_field.name == "pathogens": - # Filter to show only geographies used in indicatorsets - # You can modify this filter to show a different subset - kwargs["queryset"] = Pathogen.objects.filter( - used_in="indicatorsets" - ).order_by("display_order_number") - if db_field.name == "severity_pyramid_rungs": - # Filter to show only geographies used in indicatorsets - # You can modify this filter to show a different subset - kwargs["queryset"] = SeverityPyramidRung.objects.filter( - used_in="indicatorsets" - ).order_by("display_order_number") - return super().formfield_for_manytomany(db_field, request, **kwargs) - change_list_template = "admin/indicatorsets/indicator_set_changelist.html" def get_urls(self): @@ -104,7 +106,7 @@ def download_indicator_set(self, request): @admin.register(NonDelphiIndicatorSet) -class NonDelphiIndicatorSetAdmin(ImportExportModelAdmin): +class NonDelphiIndicatorSetAdmin(BaseIndicatorSetAdmin): """ Admin interface for the IndicatorSet model. """ @@ -167,7 +169,7 @@ def download_nondelphi_indicator_set(self, request): @admin.register(USStateIndicatorSet) -class USStateIndicatorSetAdmin(ImportExportModelAdmin): +class USStateIndicatorSetAdmin(BaseIndicatorSetAdmin): """ Admin interface for the USStateIndicatorSet model. """ From bbe25b18284c1dffbb0430639007f08cfcd21f48 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 13 Nov 2025 23:22:18 +0200 Subject: [PATCH 3/4] Renamed delphi-hosted only filter --- src/indicatorsets/forms.py | 4 ++-- src/templates/indicatorsets/indicatorSetsFilters.html | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/indicatorsets/forms.py b/src/indicatorsets/forms.py index 5d824dc..9324c11 100644 --- a/src/indicatorsets/forms.py +++ b/src/indicatorsets/forms.py @@ -54,7 +54,7 @@ class IndicatorSetFilterForm(forms.ModelForm): ) hosted_by_delphi = forms.BooleanField( - label="Hosted by Delphi", + label="Delphi-hosted Only", required=False, widget=forms.CheckboxInput(attrs={"value": "on"}), ) @@ -90,4 +90,4 @@ def __init__(self, *args, **kwargs) -> None: if field_name != "hosted_by_delphi": field.label = "" else: - field.label = "Hosted by Delphi" + field.label = "Delphi-hosted Only" diff --git a/src/templates/indicatorsets/indicatorSetsFilters.html b/src/templates/indicatorsets/indicatorSetsFilters.html index 331b8a2..f18464b 100644 --- a/src/templates/indicatorsets/indicatorSetsFilters.html +++ b/src/templates/indicatorsets/indicatorSetsFilters.html @@ -211,7 +211,7 @@
- {{ form.hosted_by_delphi|as_crispy_field }} + {{ form.temporal_scope_end|as_crispy_field }} + data-mdb-content="{{ filters_descriptions|dict_get:"temporal_scope_end" }}">
- {{ form.temporal_scope_end|as_crispy_field }} + {{ form.hosted_by_delphi|as_crispy_field }} + data-mdb-content="{{ filters_descriptions|dict_get:"hosted_by_delphi" }}">
From cd8a54b62a42bcb643bae394178e5e367cc2034e Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 13 Nov 2025 23:30:58 +0200 Subject: [PATCH 4/4] Updated settings.py (replaced few urls to google spreadsheet) --- src/epiportal/settings.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 5496aef..d7106c4 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -36,9 +36,9 @@ "other_endpoint_source_subdivisions": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=214580132", "indicator_sets": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266808975", "non_delphi_indicator_sets": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266477926", - "indicators": "https://docs.google.com/spreadsheets/d/1MdT5y1etftBQd4f1rmcZxfedi7ESSsoBhAs0cf5PCKM/export?format=csv&gid=329338228", - "other_endpoint_indicators": "https://docs.google.com/spreadsheets/d/1MdT5y1etftBQd4f1rmcZxfedi7ESSsoBhAs0cf5PCKM/export?format=csv&gid=1364181703", - "non_delphi_indicators": "https://docs.google.com/spreadsheets/d/1MdT5y1etftBQd4f1rmcZxfedi7ESSsoBhAs0cf5PCKM/export?format=csv&gid=493612863", + "indicators": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=329338228", + "other_endpoint_indicators": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1364181703", + "non_delphi_indicators": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=493612863", "us_state_indicator_sets": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=232438393", "us_state_indicators": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1158629578" }