From 5d44ff0bb5fba2559191a2efcaad0c87f5048eb4 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 26 Aug 2025 20:10:27 +0300 Subject: [PATCH 1/5] Added automated import for indicators --- src/indicatorsets/admin.py | 130 ++++++++++++++++-- src/templates/admin/change_list.html | 101 ++++++++++++++ .../indicator_set_changelist.html | 16 +++ 3 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 src/templates/admin/change_list.html create mode 100644 src/templates/admin/indicatorsets/indicator_set_changelist.html diff --git a/src/indicatorsets/admin.py b/src/indicatorsets/admin.py index 27fda14..450bd11 100644 --- a/src/indicatorsets/admin.py +++ b/src/indicatorsets/admin.py @@ -1,13 +1,19 @@ -from django.contrib import admin +import csv +import sys +from io import BytesIO, TextIOWrapper +import requests +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.urls import path +from django.utils.module_loading import import_string from import_export.admin import ImportExportModelAdmin -from indicatorsets.models import ( - IndicatorSet, - NonDelphiIndicatorSet, - FilterDescription, - ColumnDescription, -) -from indicatorsets.resources import IndicatorSetResource, NonDelphiIndicatorSetResource +from import_export.results import RowResult + +from indicatorsets.models import (ColumnDescription, FilterDescription, + IndicatorSet, NonDelphiIndicatorSet) +from indicatorsets.resources import (IndicatorSetResource, + NonDelphiIndicatorSetResource) # Register your models here. @@ -35,6 +41,60 @@ class IndicatorSetAdmin(ImportExportModelAdmin): ordering = ["name"] list_filter = ["original_data_provider"] + def get_queryset(self, request): + # Exclude proxy model objects + qs = super().get_queryset(request) + return qs.exclude(source_type="non_delphi") + + change_list_template = "admin/indicatorsets/indicator_set_changelist.html" + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-from-spreadsheet", + self.admin_site.admin_view(self.import_from_spreadsheet), + name="import_indicatorsets", + ), + ] + return custom_urls + urls + + def import_from_spreadsheet(self, request): + resource = IndicatorSetResource() + format_class = import_string("import_export.formats.base_formats.CSV") + + spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266808975" + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + self.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + self.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") + @admin.register(NonDelphiIndicatorSet) class NonDelphiIndicatorSetAdmin(ImportExportModelAdmin): @@ -61,6 +121,60 @@ class NonDelphiIndicatorSetAdmin(ImportExportModelAdmin): ordering = ["name"] list_filter = ["original_data_provider", "source_type"] + def get_queryset(self, request): + # Exclude proxy model objects + qs = super().get_queryset(request) + return qs.filter(source_type="non_delphi") + + change_list_template = "admin/indicatorsets/indicator_set_changelist.html" + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-from-spreadsheet", + self.admin_site.admin_view(self.import_from_spreadsheet), + name="import_nondelphi_indicatorsets", + ), + ] + return custom_urls + urls + + def import_from_spreadsheet(self, request): + resource = NonDelphiIndicatorSetResource() + format_class = import_string("import_export.formats.base_formats.CSV") + + spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266477926" + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + self.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + self.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") + @admin.register(FilterDescription) class FilterDescriptionAdmin(admin.ModelAdmin): diff --git a/src/templates/admin/change_list.html b/src/templates/admin/change_list.html new file mode 100644 index 0000000..feed979 --- /dev/null +++ b/src/templates/admin/change_list.html @@ -0,0 +1,101 @@ +{% extends "admin/base_site.html" %} +{% load i18n admin_urls static admin_list %} + +{% block title %}{% if cl.formset and cl.formset.errors %}{% translate "Error:" %} {% endif %}{{ block.super }}{% endblock %} +{% block extrastyle %} + {{ block.super }} + + {% if cl.formset %} + + {% endif %} + {% if cl.formset or action_form %} + + {% endif %} + {{ media.css }} + {% if not actions_on_top and not actions_on_bottom %} + + {% endif %} +{% endblock %} + +{% block extrahead %} +{{ block.super }} +{{ media.js }} + +{% endblock %} + +{% block bodyclass %}{{ block.super }} app-{{ opts.app_label }} model-{{ opts.model_name }} change-list{% endblock %} + +{% if not is_popup %} +{% block breadcrumbs %} +
+{% translate 'Home' %} +› {{ cl.opts.app_config.verbose_name }} +› {{ cl.opts.verbose_name_plural|capfirst }} +
+{% endblock %} +{% endif %} + +{% block coltype %}{% endblock %} + +{% block content %} +
+ {% block object-tools %} + + {% endblock %} + {% if cl.formset and cl.formset.errors %} +

+ {% blocktranslate count counter=cl.formset.total_error_count %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktranslate %} +

+ {{ cl.formset.non_form_errors }} + {% endif %} +
+
+ {% block filters %} + {% if cl.has_filters %} + +

{% translate 'Filter' %}

+ {% if cl.is_facets_optional or cl.has_active_filters %}
+ {% if cl.is_facets_optional %}

+ {% if cl.add_facets %}{% translate "Hide counts" %} + {% else %}{% translate "Show counts" %}{% endif %} +

{% endif %} + {% if cl.has_active_filters %}

+ ✖ {% translate "Clear all filters" %} +

{% endif %} +
{% endif %} + {% for spec in cl.filter_specs %}{% admin_list_filter cl spec %}{% endfor %} + + {% endif %} + {% endblock %} +
+ {% block search %}{% search_form cl %}{% endblock %} + {% block date_hierarchy %}{% if cl.date_hierarchy %}{% date_hierarchy cl %}{% endif %}{% endblock %} + +
{% csrf_token %} + {% if cl.formset %} +
{{ cl.formset.management_form }}
+ {% endif %} + + {% block result_list %} + {% if action_form and actions_on_top and cl.show_admin_actions %}{% admin_actions %}{% endif %} + {% result_list cl %} + {% if action_form and actions_on_bottom and cl.show_admin_actions %}{% admin_actions %}{% endif %} + {% endblock %} + {% block pagination %} +
+ {% pagination cl %} + {% if cl.formset and cl.result_count %}{% endif %} + {% endblock %} +
+
+
+
+
+
+{% endblock %} \ No newline at end of file diff --git a/src/templates/admin/indicatorsets/indicator_set_changelist.html b/src/templates/admin/indicatorsets/indicator_set_changelist.html new file mode 100644 index 0000000..2eec90e --- /dev/null +++ b/src/templates/admin/indicatorsets/indicator_set_changelist.html @@ -0,0 +1,16 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} + {{ block.super }} +
  • + {% if opts.model_name == "indicatorset" %} + + Import data from spreadsheet + + {% elif opts.model_name == "nondelphiindicatorset" %} + + Import data from spreadsheet + + {% endif %} +
    • +{% endblock %} \ No newline at end of file From 97668387f70afc61c695eb55d825983bd5fbacdf Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 27 Aug 2025 17:07:19 +0300 Subject: [PATCH 2/5] Added automated import button for the datasources and indicators --- src/datasources/admin.py | 60 +++++- src/indicators/admin.py | 188 ++++++++++++++++-- src/indicatorsets/admin.py | 2 - .../source_subdivision_changelist.html | 10 + .../indicators/indicator_changelist.html | 20 ++ 5 files changed, 259 insertions(+), 21 deletions(-) create mode 100644 src/templates/admin/datasources/source_subdivision_changelist.html create mode 100644 src/templates/admin/indicators/indicator_changelist.html diff --git a/src/datasources/admin.py b/src/datasources/admin.py index a4465ba..8b41d22 100644 --- a/src/datasources/admin.py +++ b/src/datasources/admin.py @@ -1,11 +1,16 @@ -from django.contrib import admin +from io import BytesIO, TextIOWrapper +import requests +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.urls import path +from django.utils.module_loading import import_string from import_export.admin import ImportExportModelAdmin +from import_export.results import RowResult + from datasources.models import SourceSubdivision from datasources.resources import SourceSubdivisionResource -# Register your models here. - @admin.register(SourceSubdivision) class SourceSubdivisionAdmin(ImportExportModelAdmin): @@ -26,3 +31,52 @@ class SourceSubdivisionAdmin(ImportExportModelAdmin): ordering = ["name"] list_filter = ["datasource_name"] resource_classes = [SourceSubdivisionResource] + + change_list_template = "admin/datasources/source_subdivision_changelist.html" + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-from-spreadsheet", + self.admin_site.admin_view(self.import_from_spreadsheet), + name="import_sourcesubdivisions", + ), + ] + return custom_urls + urls + + def import_from_spreadsheet(self, request): + resource = SourceSubdivisionResource() + format_class = import_string("import_export.formats.base_formats.CSV") + + spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=0" + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + self.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + self.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") diff --git a/src/indicators/admin.py b/src/indicators/admin.py index 14fdf98..13ed3c8 100644 --- a/src/indicators/admin.py +++ b/src/indicators/admin.py @@ -1,21 +1,19 @@ -from django.contrib import admin +from io import BytesIO, TextIOWrapper + +import requests +from django.contrib import admin, messages +from django.shortcuts import redirect +from django.urls import path +from django.utils.module_loading import import_string from import_export.admin import ImportExportModelAdmin +from import_export.results import RowResult -from indicators.models import ( - Category, - FormatType, - Indicator, - IndicatorGeography, - IndicatorType, - OtherEndpointIndicator, - NonDelphiIndicator -) -from indicators.resources import ( - IndicatorResource, - IndicatorBaseResource, - OtherEndpointIndicatorResource, - NonDelphiIndicatorResource, -) +from indicators.models import (Category, FormatType, Indicator, + IndicatorGeography, IndicatorType, + NonDelphiIndicator, OtherEndpointIndicator) +from indicators.resources import (IndicatorBaseResource, IndicatorResource, + NonDelphiIndicatorResource, + OtherEndpointIndicatorResource) @admin.register(IndicatorType) @@ -72,6 +70,60 @@ class IndicatorAdmin(ImportExportModelAdmin): resource_classes = [IndicatorResource, IndicatorBaseResource] + change_list_template = "admin/indicators/indicator_changelist.html" + + def get_queryset(self, request): + # Exclude proxy model objects + qs = super().get_queryset(request) + return qs.filter(source_type="covidcast") + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-from-spreadsheet", + self.admin_site.admin_view(self.import_from_spreadsheet), + name="import_indicators", + ), + ] + return custom_urls + urls + + def import_from_spreadsheet(self, request): + resource = IndicatorResource() + format_class = import_string("import_export.formats.base_formats.CSV") + + spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=329338228" + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + self.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + self.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") + @admin.register(OtherEndpointIndicator) class OtherEndpointIndicatorAdmin(ImportExportModelAdmin): @@ -93,6 +145,58 @@ class OtherEndpointIndicatorAdmin(ImportExportModelAdmin): resource_classes = [OtherEndpointIndicatorResource] + def get_queryset(self, request): + # Exclude proxy model objects + qs = super().get_queryset(request) + return qs.filter(source_type="other_endpoint") + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-from-spreadsheet", + self.admin_site.admin_view(self.import_from_spreadsheet), + name="import_otherendpoint_indicators", + ), + ] + return custom_urls + urls + + def import_from_spreadsheet(self, request): + resource = OtherEndpointIndicatorResource() + format_class = import_string("import_export.formats.base_formats.CSV") + + spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1364181703" + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + self.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + self.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") + @admin.register(NonDelphiIndicator) class NonDelphiIndicatorAdmin(ImportExportModelAdmin): @@ -110,3 +214,55 @@ class NonDelphiIndicatorAdmin(ImportExportModelAdmin): list_display_links = ("name",) resource_classes = [NonDelphiIndicatorResource] + + def get_queryset(self, request): + # Exclude proxy model objects + qs = super().get_queryset(request) + return qs.filter(source_type="non_delphi") + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-from-spreadsheet", + self.admin_site.admin_view(self.import_from_spreadsheet), + name="import_nondelphi_indicators", + ), + ] + return custom_urls + urls + + def import_from_spreadsheet(self, request): + resource = NonDelphiIndicatorResource() + format_class = import_string("import_export.formats.base_formats.CSV") + + spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=493612863" + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + self.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + self.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") diff --git a/src/indicatorsets/admin.py b/src/indicatorsets/admin.py index 450bd11..cf22258 100644 --- a/src/indicatorsets/admin.py +++ b/src/indicatorsets/admin.py @@ -1,5 +1,3 @@ -import csv -import sys from io import BytesIO, TextIOWrapper import requests diff --git a/src/templates/admin/datasources/source_subdivision_changelist.html b/src/templates/admin/datasources/source_subdivision_changelist.html new file mode 100644 index 0000000..eb02f46 --- /dev/null +++ b/src/templates/admin/datasources/source_subdivision_changelist.html @@ -0,0 +1,10 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} + {{ block.super }} +
  • + + Import data from spreadsheet + +
    • +{% endblock %} \ No newline at end of file diff --git a/src/templates/admin/indicators/indicator_changelist.html b/src/templates/admin/indicators/indicator_changelist.html new file mode 100644 index 0000000..7d421f5 --- /dev/null +++ b/src/templates/admin/indicators/indicator_changelist.html @@ -0,0 +1,20 @@ +{% extends "admin/change_list.html" %} + +{% block object-tools-items %} + {{ block.super }} +
  • + {% if opts.model_name == "indicator" %} + + Import data from spreadsheet + + {% elif opts.model_name == "otherendpointindicator" %} + + Import data from spreadsheet + + {% elif opts.model_name == "nondelphiindicator" %} + + Import data from spreadsheet + + {% endif %} +
    • +{% endblock object-tools-items %} \ No newline at end of file From a375caab4c41a961050b90668e2d1b3b44206b1f Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 27 Aug 2025 17:22:54 +0300 Subject: [PATCH 3/5] Devided SourceSubdivision into 2 separate models to handle covidcast and other endpoint data sources --- src/datasources/admin.py | 74 ++++++++++++++++++- src/datasources/models.py | 7 ++ src/datasources/resources.py | 19 ++++- .../source_subdivision_changelist.html | 8 +- 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/src/datasources/admin.py b/src/datasources/admin.py index 8b41d22..5bccca7 100644 --- a/src/datasources/admin.py +++ b/src/datasources/admin.py @@ -8,8 +8,8 @@ from import_export.admin import ImportExportModelAdmin from import_export.results import RowResult -from datasources.models import SourceSubdivision -from datasources.resources import SourceSubdivisionResource +from datasources.models import SourceSubdivision, OtherEndpointSourceSubdivision +from datasources.resources import SourceSubdivisionResource, OtherEndpointSourceSubdivisionResource @admin.register(SourceSubdivision) @@ -80,3 +80,73 @@ def import_from_spreadsheet(self, request): ) self.message_user(request, success_message, level=messages.SUCCESS) return redirect(".") + + +@admin.register(OtherEndpointSourceSubdivision) +class OtherEndpointSourceSubdivisionAdmin(ImportExportModelAdmin): + """ + Admin interface for Other Endpoint Source Subdivision model. + """ + + list_display = ( + "name", + "display_name", + "external_name", + "description", + "license", + "dua", + "datasource_name", + ) + search_fields = ("name", "display_name", "external_name") + ordering = ["name"] + list_filter = ["datasource_name"] + resource_classes = [OtherEndpointSourceSubdivisionResource] + + change_list_template = "admin/datasources/source_subdivision_changelist.html" + + def get_urls(self): + urls = super().get_urls() + custom_urls = [ + path( + "import-from-spreadsheet", + self.admin_site.admin_view(self.import_from_spreadsheet), + name="import_other_endpoint_sourcesubdivisions", + ), + ] + return custom_urls + urls + + def import_from_spreadsheet(self, request): + resource = OtherEndpointSourceSubdivisionResource() + format_class = import_string("import_export.formats.base_formats.CSV") + + spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=214580132" + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + self.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + self.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") diff --git a/src/datasources/models.py b/src/datasources/models.py index 4af68b8..676bde1 100644 --- a/src/datasources/models.py +++ b/src/datasources/models.py @@ -50,3 +50,10 @@ def __str__(self): def get_display_name(self): return self.display_name if self.display_name else self.name + + +class OtherEndpointSourceSubdivision(SourceSubdivision): + class Meta: + proxy = True + verbose_name = "Other Endpoint Source Subdivision" + verbose_name_plural = "Other Endpoint Source Subdivisions" diff --git a/src/datasources/resources.py b/src/datasources/resources.py index 766e432..7235730 100644 --- a/src/datasources/resources.py +++ b/src/datasources/resources.py @@ -1,7 +1,7 @@ from import_export import resources from import_export.fields import Field -from datasources.models import SourceSubdivision +from datasources.models import SourceSubdivision, OtherEndpointSourceSubdivision class SourceSubdivisionResource(resources.ModelResource): @@ -48,3 +48,20 @@ class Meta: "dua", "datasource_name", ) + + +class OtherEndpointSourceSubdivisionResource(SourceSubdivisionResource): + class Meta: + model = OtherEndpointSourceSubdivision + import_id_fields = ("name", "display_name") + skip_unchanged = True + report_skipped = False + fields = ( + "name", + "display_name", + "external_name", + "description", + "license", + "dua", + "datasource_name", + ) diff --git a/src/templates/admin/datasources/source_subdivision_changelist.html b/src/templates/admin/datasources/source_subdivision_changelist.html index eb02f46..1e49955 100644 --- a/src/templates/admin/datasources/source_subdivision_changelist.html +++ b/src/templates/admin/datasources/source_subdivision_changelist.html @@ -3,8 +3,14 @@ {% block object-tools-items %} {{ block.super }}
  • + {% if opts.model_name == "sourcesubdivision" %} Import data from spreadsheet + {% elif opts.model_name == "otherendpointsourcesubdivision" %} + + Import data from spreadsheet + + {% endif %}
    • -{% endblock %} \ No newline at end of file +{% endblock object-tools-items %} \ No newline at end of file From 195a786b0178601a27f74b53f2264a67820b1d5b Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 1 Sep 2025 17:19:01 +0300 Subject: [PATCH 4/5] Updated import scripts. --- src/base/utils.py | 42 ++++++ src/datasources/admin.py | 88 ++----------- ...otherendpointsourcesubdivision_and_more.py | 42 ++++++ src/datasources/models.py | 12 ++ src/datasources/resources.py | 4 + src/epiportal/settings.py | 10 ++ src/fixtures/geographic_granularities.json | 2 +- src/indicators/admin.py | 124 +++--------------- src/indicatorsets/admin.py | 82 +----------- .../0005_alter_indicatorset_source_type.py | 30 +++++ src/indicatorsets/models.py | 2 +- 11 files changed, 177 insertions(+), 261 deletions(-) create mode 100644 src/base/utils.py create mode 100644 src/datasources/migrations/0002_otherendpointsourcesubdivision_and_more.py create mode 100644 src/indicatorsets/migrations/0005_alter_indicatorset_source_type.py diff --git a/src/base/utils.py b/src/base/utils.py new file mode 100644 index 0000000..a6fdd44 --- /dev/null +++ b/src/base/utils.py @@ -0,0 +1,42 @@ +from io import BytesIO, TextIOWrapper + +import requests +from django.contrib import messages +from django.shortcuts import redirect +from django.utils.module_loading import import_string +from import_export.results import RowResult + + +def import_data(admin_instance, request, resource_class, spreadsheet_url): + resource = resource_class() + format_class = import_string("import_export.formats.base_formats.CSV") + + response = requests.get(spreadsheet_url) + response.raise_for_status() + + csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") + + dataset = format_class().create_dataset(csvfile.read()) + + result = resource.import_data(dataset, dry_run=False, raise_errors=True) + + if result.has_errors(): + error_messages = ["Import errors!"] + for error in result.base_errors: + error_messages.append(repr(error.error)) + for line, errors in result.row_errors(): + for error in errors: + error_messages.append(f"Line number: {line} - {repr(error.error)}") + admin_instance.message_user(request, "\n".join(error_messages), level=messages.ERROR) + else: + success_message = ( + "Import finished: {} new, {} updated, {} deleted and {} skipped {}." + ).format( + result.totals[RowResult.IMPORT_TYPE_NEW], + result.totals[RowResult.IMPORT_TYPE_UPDATE], + result.totals[RowResult.IMPORT_TYPE_DELETE], + result.totals[RowResult.IMPORT_TYPE_SKIP], + resource._meta.model._meta.verbose_name_plural, + ) + admin_instance.message_user(request, success_message, level=messages.SUCCESS) + return redirect(".") diff --git a/src/datasources/admin.py b/src/datasources/admin.py index 5bccca7..c41b45a 100644 --- a/src/datasources/admin.py +++ b/src/datasources/admin.py @@ -1,15 +1,13 @@ -from io import BytesIO, TextIOWrapper - -import requests -from django.contrib import admin, messages -from django.shortcuts import redirect +from django.contrib import admin from django.urls import path -from django.utils.module_loading import import_string +from django.conf import settings from import_export.admin import ImportExportModelAdmin -from import_export.results import RowResult -from datasources.models import SourceSubdivision, OtherEndpointSourceSubdivision -from datasources.resources import SourceSubdivisionResource, OtherEndpointSourceSubdivisionResource +from base.utils import import_data +from datasources.models import (OtherEndpointSourceSubdivision, + SourceSubdivision) +from datasources.resources import (OtherEndpointSourceSubdivisionResource, + SourceSubdivisionResource) @admin.register(SourceSubdivision) @@ -46,40 +44,7 @@ def get_urls(self): return custom_urls + urls def import_from_spreadsheet(self, request): - resource = SourceSubdivisionResource() - format_class = import_string("import_export.formats.base_formats.CSV") - - spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=0" - - response = requests.get(spreadsheet_url) - response.raise_for_status() - - csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") - - dataset = format_class().create_dataset(csvfile.read()) - - result = resource.import_data(dataset, dry_run=False, raise_errors=True) - - if result.has_errors(): - error_messages = ["Import errors!"] - for error in result.base_errors: - error_messages.append(repr(error.error)) - for line, errors in result.row_errors(): - for error in errors: - error_messages.append(f"Line number: {line} - {repr(error.error)}") - self.message_user(request, "\n".join(error_messages), level=messages.ERROR) - else: - success_message = ( - "Import finished: {} new, {} updated, {} deleted and {} skipped {}." - ).format( - result.totals[RowResult.IMPORT_TYPE_NEW], - result.totals[RowResult.IMPORT_TYPE_UPDATE], - result.totals[RowResult.IMPORT_TYPE_DELETE], - result.totals[RowResult.IMPORT_TYPE_SKIP], - resource._meta.model._meta.verbose_name_plural, - ) - self.message_user(request, success_message, level=messages.SUCCESS) - return redirect(".") + return import_data(self, request, SourceSubdivisionResource, settings.SPREADSHEET_URLS["source_subdivisions"]) @admin.register(OtherEndpointSourceSubdivision) @@ -116,37 +81,6 @@ def get_urls(self): return custom_urls + urls def import_from_spreadsheet(self, request): - resource = OtherEndpointSourceSubdivisionResource() - format_class = import_string("import_export.formats.base_formats.CSV") - - spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=214580132" - - response = requests.get(spreadsheet_url) - response.raise_for_status() - - csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") - - dataset = format_class().create_dataset(csvfile.read()) - - result = resource.import_data(dataset, dry_run=False, raise_errors=True) - - if result.has_errors(): - error_messages = ["Import errors!"] - for error in result.base_errors: - error_messages.append(repr(error.error)) - for line, errors in result.row_errors(): - for error in errors: - error_messages.append(f"Line number: {line} - {repr(error.error)}") - self.message_user(request, "\n".join(error_messages), level=messages.ERROR) - else: - success_message = ( - "Import finished: {} new, {} updated, {} deleted and {} skipped {}." - ).format( - result.totals[RowResult.IMPORT_TYPE_NEW], - result.totals[RowResult.IMPORT_TYPE_UPDATE], - result.totals[RowResult.IMPORT_TYPE_DELETE], - result.totals[RowResult.IMPORT_TYPE_SKIP], - resource._meta.model._meta.verbose_name_plural, - ) - self.message_user(request, success_message, level=messages.SUCCESS) - return redirect(".") + return import_data( + self, request, OtherEndpointSourceSubdivisionResource, settings.SPREADSHEET_URLS["other_endpoint_source_subdivisions"] + ) diff --git a/src/datasources/migrations/0002_otherendpointsourcesubdivision_and_more.py b/src/datasources/migrations/0002_otherendpointsourcesubdivision_and_more.py new file mode 100644 index 0000000..25da00b --- /dev/null +++ b/src/datasources/migrations/0002_otherendpointsourcesubdivision_and_more.py @@ -0,0 +1,42 @@ +# Generated by Django 5.2.5 on 2025-08-27 14:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("datasources", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="OtherEndpointSourceSubdivision", + fields=[], + options={ + "verbose_name": "Other Endpoint Source Subdivision", + "verbose_name_plural": "Other Endpoint Source Subdivisions", + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("datasources.sourcesubdivision",), + ), + migrations.AddField( + model_name="sourcesubdivision", + name="source_type", + field=models.CharField( + blank=True, + choices=[ + ("covidcast", "Covidcast"), + ("other_endpoint", "Other Endpoint"), + ("non_delphi", "Non Delphi"), + ], + default="covidcast", + help_text="Type of source for the source subdivision", + max_length=255, + null=True, + verbose_name="Source Type", + ), + ), + ] diff --git a/src/datasources/models.py b/src/datasources/models.py index 676bde1..5492efc 100644 --- a/src/datasources/models.py +++ b/src/datasources/models.py @@ -1,5 +1,7 @@ from django.db import models +from base.models import SOURCE_TYPES + # Create your models here. class SourceSubdivision(models.Model): @@ -32,6 +34,16 @@ class SourceSubdivision(models.Model): verbose_name="Datasource Name", max_length=255, blank=True ) + source_type: models.CharField = models.CharField( + verbose_name="Source Type", + max_length=255, + choices=SOURCE_TYPES, + default="covidcast", + help_text="Type of source for the source subdivision", + blank=True, + null=True, + ) + class Meta: ordering = ["name"] verbose_name = "Source Subdivision" diff --git a/src/datasources/resources.py b/src/datasources/resources.py index 7235730..b25dc41 100644 --- a/src/datasources/resources.py +++ b/src/datasources/resources.py @@ -65,3 +65,7 @@ class Meta: "dua", "datasource_name", ) + + def after_save_instance(self, instance, row, **kwargs): + instance.source_type = "other_endpoint" + instance.save() diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index ce38745..ec824ad 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -31,6 +31,16 @@ EPIDATA_URL = os.environ.get("EPIDATA_URL", "https://api.delphi.cmu.edu/epidata/") EPIDATA_API_KEY = os.environ.get("EPIDATA_API_KEY", "") +SPREADSHEET_URLS = { + "source_subdivisions": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=0", + "other_endpoint_source_subdivisions": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=214580132", + "indicator_sets": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266808975", + "non_delphi_indicator_sets": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266477926", + "indicators": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=329338228", + "other_endpoint_indicators": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1364181703", + "non_delphi_indicators": "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=493612863" +} + SENTRY_DSN = os.environ.get('SENTRY_DSN') if SENTRY_DSN: sentry_sdk.init( diff --git a/src/fixtures/geographic_granularities.json b/src/fixtures/geographic_granularities.json index 6c4b347..9e2cd51 100644 --- a/src/fixtures/geographic_granularities.json +++ b/src/fixtures/geographic_granularities.json @@ -14,7 +14,7 @@ "model": "base.Geography", "pk": 2, "fields": { - "name": "hhs", + "name": "hhs-region", "display_name": "U.S. HHS Region", "display_order_number": 2, "used_in": "indicatorsets", diff --git a/src/indicators/admin.py b/src/indicators/admin.py index 13ed3c8..41c6e16 100644 --- a/src/indicators/admin.py +++ b/src/indicators/admin.py @@ -1,13 +1,9 @@ -from io import BytesIO, TextIOWrapper - -import requests -from django.contrib import admin, messages -from django.shortcuts import redirect +from django.conf import settings +from django.contrib import admin from django.urls import path -from django.utils.module_loading import import_string from import_export.admin import ImportExportModelAdmin -from import_export.results import RowResult +from base.utils import import_data from indicators.models import (Category, FormatType, Indicator, IndicatorGeography, IndicatorType, NonDelphiIndicator, OtherEndpointIndicator) @@ -89,40 +85,9 @@ def get_urls(self): return custom_urls + urls def import_from_spreadsheet(self, request): - resource = IndicatorResource() - format_class = import_string("import_export.formats.base_formats.CSV") - - spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=329338228" - - response = requests.get(spreadsheet_url) - response.raise_for_status() - - csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") - - dataset = format_class().create_dataset(csvfile.read()) - - result = resource.import_data(dataset, dry_run=False, raise_errors=True) - - if result.has_errors(): - error_messages = ["Import errors!"] - for error in result.base_errors: - error_messages.append(repr(error.error)) - for line, errors in result.row_errors(): - for error in errors: - error_messages.append(f"Line number: {line} - {repr(error.error)}") - self.message_user(request, "\n".join(error_messages), level=messages.ERROR) - else: - success_message = ( - "Import finished: {} new, {} updated, {} deleted and {} skipped {}." - ).format( - result.totals[RowResult.IMPORT_TYPE_NEW], - result.totals[RowResult.IMPORT_TYPE_UPDATE], - result.totals[RowResult.IMPORT_TYPE_DELETE], - result.totals[RowResult.IMPORT_TYPE_SKIP], - resource._meta.model._meta.verbose_name_plural, - ) - self.message_user(request, success_message, level=messages.SUCCESS) - return redirect(".") + return import_data( + self, request, IndicatorResource, settings.SPREADSHEET_URLS["indicators"] + ) @admin.register(OtherEndpointIndicator) @@ -145,6 +110,8 @@ class OtherEndpointIndicatorAdmin(ImportExportModelAdmin): resource_classes = [OtherEndpointIndicatorResource] + change_list_template = "admin/indicators/indicator_changelist.html" + def get_queryset(self, request): # Exclude proxy model objects qs = super().get_queryset(request) @@ -162,40 +129,12 @@ def get_urls(self): return custom_urls + urls def import_from_spreadsheet(self, request): - resource = OtherEndpointIndicatorResource() - format_class = import_string("import_export.formats.base_formats.CSV") - - spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1364181703" - - response = requests.get(spreadsheet_url) - response.raise_for_status() - - csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") - - dataset = format_class().create_dataset(csvfile.read()) - - result = resource.import_data(dataset, dry_run=False, raise_errors=True) - - if result.has_errors(): - error_messages = ["Import errors!"] - for error in result.base_errors: - error_messages.append(repr(error.error)) - for line, errors in result.row_errors(): - for error in errors: - error_messages.append(f"Line number: {line} - {repr(error.error)}") - self.message_user(request, "\n".join(error_messages), level=messages.ERROR) - else: - success_message = ( - "Import finished: {} new, {} updated, {} deleted and {} skipped {}." - ).format( - result.totals[RowResult.IMPORT_TYPE_NEW], - result.totals[RowResult.IMPORT_TYPE_UPDATE], - result.totals[RowResult.IMPORT_TYPE_DELETE], - result.totals[RowResult.IMPORT_TYPE_SKIP], - resource._meta.model._meta.verbose_name_plural, - ) - self.message_user(request, success_message, level=messages.SUCCESS) - return redirect(".") + return import_data( + self, + request, + OtherEndpointIndicatorResource, + settings.SPREADSHEET_URLS["other_endpoint_indicators"], + ) @admin.register(NonDelphiIndicator) @@ -215,6 +154,8 @@ class NonDelphiIndicatorAdmin(ImportExportModelAdmin): resource_classes = [NonDelphiIndicatorResource] + change_list_template = "admin/indicators/indicator_changelist.html" + def get_queryset(self, request): # Exclude proxy model objects qs = super().get_queryset(request) @@ -232,37 +173,6 @@ def get_urls(self): return custom_urls + urls def import_from_spreadsheet(self, request): - resource = NonDelphiIndicatorResource() - format_class = import_string("import_export.formats.base_formats.CSV") - spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=493612863" - response = requests.get(spreadsheet_url) - response.raise_for_status() - - csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") - - dataset = format_class().create_dataset(csvfile.read()) - - result = resource.import_data(dataset, dry_run=False, raise_errors=True) - - if result.has_errors(): - error_messages = ["Import errors!"] - for error in result.base_errors: - error_messages.append(repr(error.error)) - for line, errors in result.row_errors(): - for error in errors: - error_messages.append(f"Line number: {line} - {repr(error.error)}") - self.message_user(request, "\n".join(error_messages), level=messages.ERROR) - else: - success_message = ( - "Import finished: {} new, {} updated, {} deleted and {} skipped {}." - ).format( - result.totals[RowResult.IMPORT_TYPE_NEW], - result.totals[RowResult.IMPORT_TYPE_UPDATE], - result.totals[RowResult.IMPORT_TYPE_DELETE], - result.totals[RowResult.IMPORT_TYPE_SKIP], - resource._meta.model._meta.verbose_name_plural, - ) - self.message_user(request, success_message, level=messages.SUCCESS) - return redirect(".") + return import_data(self, request, NonDelphiIndicatorResource, spreadsheet_url) diff --git a/src/indicatorsets/admin.py b/src/indicatorsets/admin.py index cf22258..f8819ec 100644 --- a/src/indicatorsets/admin.py +++ b/src/indicatorsets/admin.py @@ -1,13 +1,9 @@ -from io import BytesIO, TextIOWrapper - -import requests -from django.contrib import admin, messages -from django.shortcuts import redirect +from django.contrib import admin from django.urls import path -from django.utils.module_loading import import_string +from django.conf import settings from import_export.admin import ImportExportModelAdmin -from import_export.results import RowResult +from base.utils import import_data from indicatorsets.models import (ColumnDescription, FilterDescription, IndicatorSet, NonDelphiIndicatorSet) from indicatorsets.resources import (IndicatorSetResource, @@ -58,40 +54,7 @@ def get_urls(self): return custom_urls + urls def import_from_spreadsheet(self, request): - resource = IndicatorSetResource() - format_class = import_string("import_export.formats.base_formats.CSV") - - spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266808975" - - response = requests.get(spreadsheet_url) - response.raise_for_status() - - csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") - - dataset = format_class().create_dataset(csvfile.read()) - - result = resource.import_data(dataset, dry_run=False, raise_errors=True) - - if result.has_errors(): - error_messages = ["Import errors!"] - for error in result.base_errors: - error_messages.append(repr(error.error)) - for line, errors in result.row_errors(): - for error in errors: - error_messages.append(f"Line number: {line} - {repr(error.error)}") - self.message_user(request, "\n".join(error_messages), level=messages.ERROR) - else: - success_message = ( - "Import finished: {} new, {} updated, {} deleted and {} skipped {}." - ).format( - result.totals[RowResult.IMPORT_TYPE_NEW], - result.totals[RowResult.IMPORT_TYPE_UPDATE], - result.totals[RowResult.IMPORT_TYPE_DELETE], - result.totals[RowResult.IMPORT_TYPE_SKIP], - resource._meta.model._meta.verbose_name_plural, - ) - self.message_user(request, success_message, level=messages.SUCCESS) - return redirect(".") + return import_data(self, request, IndicatorSetResource, settings.SPREADSHEET_URLS["indicator_sets"]) @admin.register(NonDelphiIndicatorSet) @@ -138,40 +101,9 @@ def get_urls(self): return custom_urls + urls def import_from_spreadsheet(self, request): - resource = NonDelphiIndicatorSetResource() - format_class = import_string("import_export.formats.base_formats.CSV") - - spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=1266477926" - - response = requests.get(spreadsheet_url) - response.raise_for_status() - - csvfile = TextIOWrapper(BytesIO(response.content), encoding="utf-8") - - dataset = format_class().create_dataset(csvfile.read()) - - result = resource.import_data(dataset, dry_run=False, raise_errors=True) - - if result.has_errors(): - error_messages = ["Import errors!"] - for error in result.base_errors: - error_messages.append(repr(error.error)) - for line, errors in result.row_errors(): - for error in errors: - error_messages.append(f"Line number: {line} - {repr(error.error)}") - self.message_user(request, "\n".join(error_messages), level=messages.ERROR) - else: - success_message = ( - "Import finished: {} new, {} updated, {} deleted and {} skipped {}." - ).format( - result.totals[RowResult.IMPORT_TYPE_NEW], - result.totals[RowResult.IMPORT_TYPE_UPDATE], - result.totals[RowResult.IMPORT_TYPE_DELETE], - result.totals[RowResult.IMPORT_TYPE_SKIP], - resource._meta.model._meta.verbose_name_plural, - ) - self.message_user(request, success_message, level=messages.SUCCESS) - return redirect(".") + return import_data( + self, request, NonDelphiIndicatorSetResource, settings.SPREADSHEET_URLS["non_delphi_indicator_sets"] + ) @admin.register(FilterDescription) diff --git a/src/indicatorsets/migrations/0005_alter_indicatorset_source_type.py b/src/indicatorsets/migrations/0005_alter_indicatorset_source_type.py new file mode 100644 index 0000000..d923c86 --- /dev/null +++ b/src/indicatorsets/migrations/0005_alter_indicatorset_source_type.py @@ -0,0 +1,30 @@ +# Generated by Django 5.2.5 on 2025-08-27 14:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("indicatorsets", "0004_columndescription_filterdescription"), + ] + + operations = [ + migrations.AlterField( + model_name="indicatorset", + name="source_type", + field=models.CharField( + blank=True, + choices=[ + ("covidcast", "Covidcast"), + ("other_endpoint", "Other Endpoint"), + ("non_delphi", "Non Delphi"), + ], + default="covidcast", + help_text="Type of source for the indicator set", + max_length=255, + null=True, + verbose_name="Source Type", + ), + ), + ] diff --git a/src/indicatorsets/models.py b/src/indicatorsets/models.py index ac00c76..a39d2a5 100644 --- a/src/indicatorsets/models.py +++ b/src/indicatorsets/models.py @@ -190,7 +190,7 @@ class IndicatorSet(models.Model): max_length=255, choices=SOURCE_TYPES, default="covidcast", - help_text="Type of source for the indicator", + help_text="Type of source for the indicator set", blank=True, null=True, ) From 8d00dc815801af7c0bbaee10c1cb8fb1bd0e72d0 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 1 Sep 2025 23:48:18 +0300 Subject: [PATCH 5/5] Added button to download source file --- src/base/utils.py | 12 ++++- src/datasources/admin.py | 44 ++++++++++++++--- src/indicators/admin.py | 36 +++++++++++++- src/indicatorsets/admin.py | 48 +++++++++++++++---- .../source_subdivision_changelist.html | 6 +++ .../indicators/indicator_changelist.html | 9 ++++ .../indicator_set_changelist.html | 6 +++ 7 files changed, 144 insertions(+), 17 deletions(-) diff --git a/src/base/utils.py b/src/base/utils.py index a6fdd44..caaa9c5 100644 --- a/src/base/utils.py +++ b/src/base/utils.py @@ -4,6 +4,7 @@ from django.contrib import messages from django.shortcuts import redirect from django.utils.module_loading import import_string +from django.http import FileResponse from import_export.results import RowResult @@ -27,7 +28,9 @@ def import_data(admin_instance, request, resource_class, spreadsheet_url): for line, errors in result.row_errors(): for error in errors: error_messages.append(f"Line number: {line} - {repr(error.error)}") - admin_instance.message_user(request, "\n".join(error_messages), level=messages.ERROR) + admin_instance.message_user( + request, "\n".join(error_messages), level=messages.ERROR + ) else: success_message = ( "Import finished: {} new, {} updated, {} deleted and {} skipped {}." @@ -40,3 +43,10 @@ def import_data(admin_instance, request, resource_class, spreadsheet_url): ) admin_instance.message_user(request, success_message, level=messages.SUCCESS) return redirect(".") + + +def download_source_file(url, file_name): + response = requests.get(url) + response.raise_for_status() + return FileResponse(BytesIO(response.content), as_attachment=True, filename=file_name) + diff --git a/src/datasources/admin.py b/src/datasources/admin.py index c41b45a..e4df1da 100644 --- a/src/datasources/admin.py +++ b/src/datasources/admin.py @@ -3,11 +3,12 @@ from django.conf import settings from import_export.admin import ImportExportModelAdmin -from base.utils import import_data -from datasources.models import (OtherEndpointSourceSubdivision, - SourceSubdivision) -from datasources.resources import (OtherEndpointSourceSubdivisionResource, - SourceSubdivisionResource) +from base.utils import import_data, download_source_file +from datasources.models import OtherEndpointSourceSubdivision, SourceSubdivision +from datasources.resources import ( + OtherEndpointSourceSubdivisionResource, + SourceSubdivisionResource, +) @admin.register(SourceSubdivision) @@ -40,11 +41,26 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_spreadsheet), name="import_sourcesubdivisions", ), + path( + "download-source-file", + self.admin_site.admin_view(self.download_source_subdivision), + name="download_source_subdivision", + ), ] return custom_urls + urls def import_from_spreadsheet(self, request): - return import_data(self, request, SourceSubdivisionResource, settings.SPREADSHEET_URLS["source_subdivisions"]) + return import_data( + self, + request, + SourceSubdivisionResource, + settings.SPREADSHEET_URLS["source_subdivisions"], + ) + + def download_source_subdivision(self, request): + return download_source_file( + settings.SPREADSHEET_URLS["source_subdivisions"], "Source_Subdivisions.csv" + ) @admin.register(OtherEndpointSourceSubdivision) @@ -77,10 +93,24 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_spreadsheet), name="import_other_endpoint_sourcesubdivisions", ), + path( + "download-source-file", + self.admin_site.admin_view(self.download_other_endpoint_sourcesubdivision), + name="download_other_endpoint_sourcesubdivision", + ), ] return custom_urls + urls def import_from_spreadsheet(self, request): return import_data( - self, request, OtherEndpointSourceSubdivisionResource, settings.SPREADSHEET_URLS["other_endpoint_source_subdivisions"] + self, + request, + OtherEndpointSourceSubdivisionResource, + settings.SPREADSHEET_URLS["other_endpoint_source_subdivisions"], + ) + + def download_other_endpoint_sourcesubdivision(self, request): + return download_source_file( + settings.SPREADSHEET_URLS["other_endpoint_source_subdivisions"], + "Other_Endpoint_Source_Subdivisions.csv", ) diff --git a/src/indicators/admin.py b/src/indicators/admin.py index 41c6e16..091d3f8 100644 --- a/src/indicators/admin.py +++ b/src/indicators/admin.py @@ -3,7 +3,7 @@ from django.urls import path from import_export.admin import ImportExportModelAdmin -from base.utils import import_data +from base.utils import download_source_file, import_data from indicators.models import (Category, FormatType, Indicator, IndicatorGeography, IndicatorType, NonDelphiIndicator, OtherEndpointIndicator) @@ -81,6 +81,11 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_spreadsheet), name="import_indicators", ), + path( + "download-source-file", + self.admin_site.admin_view(self.download_indicator), + name="download_indicator", + ), ] return custom_urls + urls @@ -89,6 +94,11 @@ def import_from_spreadsheet(self, request): self, request, IndicatorResource, settings.SPREADSHEET_URLS["indicators"] ) + def download_indicator(self, request): + return download_source_file( + settings.SPREADSHEET_URLS["indicators"], "Indicators.csv" + ) + @admin.register(OtherEndpointIndicator) class OtherEndpointIndicatorAdmin(ImportExportModelAdmin): @@ -125,6 +135,13 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_spreadsheet), name="import_otherendpoint_indicators", ), + path( + "download-source-file", + self.admin_site.admin_view( + self.download_other_endpoint_indicator + ), + name="download_other_endpoint_indicator", + ), ] return custom_urls + urls @@ -136,6 +153,12 @@ def import_from_spreadsheet(self, request): settings.SPREADSHEET_URLS["other_endpoint_indicators"], ) + def download_other_endpoint_indicator(self, request): + return download_source_file( + settings.SPREADSHEET_URLS["other_endpoint_indicators"], + "Other_Endpoint_Indicators.csv", + ) + @admin.register(NonDelphiIndicator) class NonDelphiIndicatorAdmin(ImportExportModelAdmin): @@ -169,6 +192,11 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_spreadsheet), name="import_nondelphi_indicators", ), + path( + "download-source-file", + self.admin_site.admin_view(self.download_nondelphi_indicator), + name="download_nondelphi_indicator", + ), ] return custom_urls + urls @@ -176,3 +204,9 @@ def import_from_spreadsheet(self, request): spreadsheet_url = "https://docs.google.com/spreadsheets/d/1zb7ItJzY5oq1n-2xtvnPBiJu2L3AqmCKubrLkKJZVHs/export?format=csv&gid=493612863" return import_data(self, request, NonDelphiIndicatorResource, spreadsheet_url) + + def download_nondelphi_indicator(self, request): + return download_source_file( + settings.SPREADSHEET_URLS["non_delphi_indicators"], + "Non_Delphi_Indicators.csv", + ) diff --git a/src/indicatorsets/admin.py b/src/indicatorsets/admin.py index f8819ec..0072d02 100644 --- a/src/indicatorsets/admin.py +++ b/src/indicatorsets/admin.py @@ -1,13 +1,16 @@ +from django.conf import settings from django.contrib import admin from django.urls import path -from django.conf import settings from import_export.admin import ImportExportModelAdmin -from base.utils import import_data -from indicatorsets.models import (ColumnDescription, FilterDescription, - IndicatorSet, NonDelphiIndicatorSet) -from indicatorsets.resources import (IndicatorSetResource, - NonDelphiIndicatorSetResource) +from base.utils import download_source_file, import_data +from indicatorsets.models import ( + ColumnDescription, + FilterDescription, + IndicatorSet, + NonDelphiIndicatorSet, +) +from indicatorsets.resources import IndicatorSetResource, NonDelphiIndicatorSetResource # Register your models here. @@ -50,11 +53,26 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_spreadsheet), name="import_indicatorsets", ), + path( + "download-source-file", + self.admin_site.admin_view(self.download_indicator_set), + name="download_indicator_set", + ), ] return custom_urls + urls def import_from_spreadsheet(self, request): - return import_data(self, request, IndicatorSetResource, settings.SPREADSHEET_URLS["indicator_sets"]) + return import_data( + self, + request, + IndicatorSetResource, + settings.SPREADSHEET_URLS["indicator_sets"], + ) + + def download_indicator_set(self, request): + return download_source_file( + settings.SPREADSHEET_URLS["indicator_sets"], "Indicator_Sets.csv" + ) @admin.register(NonDelphiIndicatorSet) @@ -97,12 +115,26 @@ def get_urls(self): self.admin_site.admin_view(self.import_from_spreadsheet), name="import_nondelphi_indicatorsets", ), + path( + "download-source-file", + self.admin_site.admin_view(self.download_nondelphi_indicator_set), + name="download_nondelphi_indicator_set", + ), ] return custom_urls + urls def import_from_spreadsheet(self, request): return import_data( - self, request, NonDelphiIndicatorSetResource, settings.SPREADSHEET_URLS["non_delphi_indicator_sets"] + self, + request, + NonDelphiIndicatorSetResource, + settings.SPREADSHEET_URLS["non_delphi_indicator_sets"], + ) + + def download_nondelphi_indicator_set(self, request): + return download_source_file( + settings.SPREADSHEET_URLS["non_delphi_indicator_sets"], + "Non_Delphi_Indicator_Sets.csv", ) diff --git a/src/templates/admin/datasources/source_subdivision_changelist.html b/src/templates/admin/datasources/source_subdivision_changelist.html index 1e49955..ea087a4 100644 --- a/src/templates/admin/datasources/source_subdivision_changelist.html +++ b/src/templates/admin/datasources/source_subdivision_changelist.html @@ -7,10 +7,16 @@ Import data from spreadsheet + + Download source file + {% elif opts.model_name == "otherendpointsourcesubdivision" %} Import data from spreadsheet + + Download source file + {% endif %} {% endblock object-tools-items %} \ No newline at end of file diff --git a/src/templates/admin/indicators/indicator_changelist.html b/src/templates/admin/indicators/indicator_changelist.html index 7d421f5..e3af998 100644 --- a/src/templates/admin/indicators/indicator_changelist.html +++ b/src/templates/admin/indicators/indicator_changelist.html @@ -7,14 +7,23 @@ Import data from spreadsheet + + Download source file + {% elif opts.model_name == "otherendpointindicator" %} Import data from spreadsheet + + Download source file + {% elif opts.model_name == "nondelphiindicator" %} Import data from spreadsheet + + Download source file + {% endif %} {% endblock object-tools-items %} \ No newline at end of file diff --git a/src/templates/admin/indicatorsets/indicator_set_changelist.html b/src/templates/admin/indicatorsets/indicator_set_changelist.html index 2eec90e..c655611 100644 --- a/src/templates/admin/indicatorsets/indicator_set_changelist.html +++ b/src/templates/admin/indicatorsets/indicator_set_changelist.html @@ -7,10 +7,16 @@ Import data from spreadsheet + + Download source file + {% elif opts.model_name == "nondelphiindicatorset" %} Import data from spreadsheet + + Download source file + {% endif %} {% endblock %} \ No newline at end of file