From b6111fbf321c491b5f0e178935a79c3a4e1ed43e Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 24 Jul 2025 20:51:42 +0300 Subject: [PATCH 01/49] Added app version. Fixed fluview_clinical indicators (before this app tried to use endpoint to fluview_clinical indicators instead of using --- src/epiportal/settings.py | 2 ++ src/indicatorsets/views.py | 3 ++- src/templates/indicatorsets/indicatorSetsFilters.html | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 661afc1..7bd1db8 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,6 +24,8 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration +APP_VERSION = "1.0.0" + EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") EPIDATA_URL = os.environ.get("EPIDATA_URL", "https://api.delphi.cmu.edu/epidata/") diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index 760ddd1..7087882 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -184,6 +184,7 @@ def get_context_data(self, **kwargs): context["epidata_url"] = settings.EPIDATA_URL context["form"] = IndicatorSetFilterForm(initial=url_params_dict) context["filter"] = filter + context["APP_VERSION"] = settings.APP_VERSION context["indicator_sets"] = filter.qs.annotate( is_ongoing=Case( When( @@ -264,7 +265,7 @@ def epivis(request): indicator["indicator"], indicator["indicator"] ), "params": { - "_endpoint": indicator["_endpoint"], + "_endpoint": indicator["_endpoint"] if indicator["data_source"] == "fluview" else "fluview_clinical", "regions": geo["id"], "custom_title": generate_epivis_custom_title( indicator, geo["text"] diff --git a/src/templates/indicatorsets/indicatorSetsFilters.html b/src/templates/indicatorsets/indicatorSetsFilters.html index cfeb34b..606640e 100644 --- a/src/templates/indicatorsets/indicatorSetsFilters.html +++ b/src/templates/indicatorsets/indicatorSetsFilters.html @@ -276,6 +276,9 @@ Other Feedback or Suggestion +
+ v {{ APP_VERSION }} +
From cb542f12264203bed836cabcfdeb4edc06285f57 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 25 Jul 2025 00:42:32 +0300 Subject: [PATCH 02/49] Changed different location hint message --- src/assets/js/selectedIndicatorsModal.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index 51c799d..c45bfbc 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -233,7 +233,8 @@ $("#showSelectedIndicatorsButton").click(function () { }); nonCovidcastIndicatorSets = [...new Set(checkedIndicatorMembers.filter(indicator => indicator["_endpoint"] != "covidcast").map((indicator) => indicator["indicator_set"]))]; var otherEndpointLocationsWarning = `` if (indicatorHandler.getFluviewIndicators().length > 0) { $("#differentLocationNote").html(otherEndpointLocationsWarning) From 153b3f866867d331dda1cafc08eeec70df99f1a5 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 28 Jul 2025 10:59:29 +0300 Subject: [PATCH 03/49] Fixed carrot in location search dropdown --- src/assets/css/style.css | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index 4877570..d600578 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -1314,17 +1314,12 @@ h6 { .select2 { display: block; width: 100%!important; - padding: 0.375rem 2.25rem 0.375rem 0.75rem; + padding: 0.375rem 0.5rem 0.375rem 0.75rem; -moz-padding-start: calc(0.75rem - 3px); font-size: 1rem; font-weight: 400; line-height: 1.5; color: #212529; - background-color: #fff; - background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e"); - background-repeat: no-repeat; - background-position: right 0.75rem center; - background-size: 16px 12px; border: 1px solid #ced4da; border-radius: 0.25rem; transition: border-color 0.15s ease-in-out, box-shadow 0.15s ease-in-out; @@ -1339,6 +1334,25 @@ h6 { .select2-container--default .select2-selection--multiple { border: none!important; + position: relative; + padding-right: 20px; +} + +.select2-container--default .select2-selection--multiple:after { + content: ''; + border-color: #888 transparent transparent transparent; + border-style: solid; + border-width: 5px 4px 0 4px; + position: absolute; + top: 50%; + right: 5px; + transform: translateY(-50%); + pointer-events: none; +} + +.select2-container--default.select2-container--open .select2-selection--multiple:after { + border-color: transparent transparent #888 transparent; + border-width: 0 4px 5px 4px; } .select2-container--default .select2-selection--single .select2-selection__arrow { From 69f2666fea3a0b2768ce0f403b2de9c2ad48fa97 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 28 Jul 2025 16:31:03 +0300 Subject: [PATCH 04/49] Added geographic levels ordering in indicator set table --- src/indicatorsets/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/indicatorsets/models.py b/src/indicatorsets/models.py index 1e9a0e4..ac00c76 100644 --- a/src/indicatorsets/models.py +++ b/src/indicatorsets/models.py @@ -214,7 +214,7 @@ def __str__(self): @property def get_geographic_levels(self): - return [geo.display_name for geo in self.geographic_levels.all()] + return [geo.display_name for geo in self.geographic_levels.all().order_by("display_order_number")] class NonDelphiIndicatorSet(IndicatorSet): From 6dd2077d30aa28d4bc1420268844a44f268f8faa Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 29 Jul 2025 01:42:34 +0300 Subject: [PATCH 05/49] Changed alternate row color --- src/assets/css/custom_styles.css | 29 +++++++++++++------ src/assets/js/indicatorSetsTable.js | 7 ++++- src/templates/index.html | 5 ++-- .../indicatorsets/indicatorSetsTable.html | 2 +- 4 files changed, 30 insertions(+), 13 deletions(-) diff --git a/src/assets/css/custom_styles.css b/src/assets/css/custom_styles.css index e8faf2e..5936ee1 100644 --- a/src/assets/css/custom_styles.css +++ b/src/assets/css/custom_styles.css @@ -32,7 +32,7 @@ border: 1px solid #ddd; border-radius: 4px; padding: 15px; - font-family: 'Courier New', Courier, monospace; + font-family: "Courier New", Courier, monospace; font-size: 14px; line-height: 1.6; overflow-x: auto; @@ -41,13 +41,11 @@ } .highlight-code { - font-size: 0.875em!important; - color: var(--mdb-code-color)!important; - word-wrap: break-word!important; + font-size: 0.875em !important; + color: var(--mdb-code-color) !important; + word-wrap: break-word !important; } - - /* Loader Overlay */ .loader-overlay { position: fixed; @@ -159,10 +157,23 @@ } td.dt-empty { - text-align: left!important; + text-align: left !important; } .margin-left-right-auto { - margin-left: auto!important; - margin-right: auto!important; + margin-left: auto !important; + margin-right: auto !important; +} + +.table-striped > tbody > tr.odd-row { + background-color: rgb(230, 230, 230) !important; +} + + +table.dataTable tbody tr>.dtfc-fixed-start, table.dataTable tbody tr>.dtfc-fixed-end { + background-color: transparent !important; /* Match the table header background */ +} + +tr.transparent-background { + background-color: transparent !important; } \ No newline at end of file diff --git a/src/assets/js/indicatorSetsTable.js b/src/assets/js/indicatorSetsTable.js index 3d73836..40d47ec 100644 --- a/src/assets/js/indicatorSetsTable.js +++ b/src/assets/js/indicatorSetsTable.js @@ -33,7 +33,12 @@ var table = new DataTable("#indicatorSetsTable", { smart: true, highlight: true, }, - sDom: 'ltipr' + sDom: 'ltipr', + rowCallback: function(row, data, index) { + if (index % 2 === 0) { + $(row).addClass('odd-row'); + } + } }); // new DataTable.Buttons(table, { diff --git a/src/templates/index.html b/src/templates/index.html index 3f4ab30..7da44d7 100644 --- a/src/templates/index.html +++ b/src/templates/index.html @@ -21,8 +21,6 @@ - - @@ -40,6 +38,9 @@ href="https://cdn.datatables.net/v/dt/dt-2.1.8/b-3.2.0/b-colvis-3.2.0/cr-2.0.4/fc-5.0.4/fh-4.0.1/kt-2.12.1/r-3.0.3/rg-1.5.1/rr-1.5.0/sc-2.4.3/sp-2.3.3/sl-2.1.0/datatables.min.css" /> + + + diff --git a/src/templates/indicatorsets/indicatorSetsTable.html b/src/templates/indicatorsets/indicatorSetsTable.html index 96af4d7..21fbe89 100644 --- a/src/templates/indicatorsets/indicatorSetsTable.html +++ b/src/templates/indicatorsets/indicatorSetsTable.html @@ -10,7 +10,7 @@ href="https://cdn.datatables.net/plug-ins/1.10.13/features/mark.js/datatables.mark.min.css">
- +
From 263bc5155b9de8359a49fb64af79ba674e5d0ffa Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 29 Jul 2025 12:56:30 +0300 Subject: [PATCH 06/49] Fixed location search filter (after deselecting indicators). Updated app version. --- src/assets/js/selectedIndicatorsModal.js | 4 ++++ src/epiportal/settings.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index c45bfbc..27b11b9 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -212,6 +212,10 @@ $("#showSelectedIndicatorsButton").click(function () { alertPlaceholder.innerHTML = ""; const availableGeos = getAvailableGeos(checkedIndicatorMembers); const locationIds = $("#location_search").select2("data").map((item) => item.id); + + if ($('#geographic_value').hasClass("select2-hidden-accessible")) { + $('#geographic_value').select2('destroy').empty(); + } $("#geographic_value").select2({ data: availableGeos, minimumInputLength: 0, diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 7bd1db8..7fd4660 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.0" +APP_VERSION = "1.0.1" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From b7b2c12d0915582a150888c816d44153c233140f Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 29 Jul 2025 18:33:17 +0300 Subject: [PATCH 07/49] Fixed transparent background for first two rows in indicator sets table --- src/assets/css/custom_styles.css | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/assets/css/custom_styles.css b/src/assets/css/custom_styles.css index 5936ee1..82b1c97 100644 --- a/src/assets/css/custom_styles.css +++ b/src/assets/css/custom_styles.css @@ -169,11 +169,12 @@ td.dt-empty { background-color: rgb(230, 230, 230) !important; } - -table.dataTable tbody tr>.dtfc-fixed-start, table.dataTable tbody tr>.dtfc-fixed-end { - background-color: transparent !important; /* Match the table header background */ +.odd-row > .dtfc-fixed-start, +.odd-row > .dtfc-fixed-end { + background-color: rgb(230, 230, 230) !important; } -tr.transparent-background { - background-color: transparent !important; +tr:not(.odd-row) > .dtfc-fixed-start, +tr:not(.odd-row) > .dtfc-fixed-end { + background-color: #f6f9ff !important; } \ No newline at end of file From 02ce81f8f795f98a859bac4c76014e5224161147 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 29 Jul 2025 19:23:18 +0300 Subject: [PATCH 08/49] Fixed location dropbox filtering & preserving previously selected values --- src/assets/css/custom_styles.css | 71 +++++++++++++++++++++++- src/assets/js/selectedIndicatorsModal.js | 59 ++++++++++++-------- src/epiportal/settings.py | 2 +- 3 files changed, 108 insertions(+), 24 deletions(-) diff --git a/src/assets/css/custom_styles.css b/src/assets/css/custom_styles.css index 82b1c97..1466579 100644 --- a/src/assets/css/custom_styles.css +++ b/src/assets/css/custom_styles.css @@ -177,4 +177,73 @@ td.dt-empty { tr:not(.odd-row) > .dtfc-fixed-start, tr:not(.odd-row) > .dtfc-fixed-end { background-color: #f6f9ff !important; -} \ No newline at end of file +} + + +#geo-loader { + display: flex; + align-items: center; + justify-content: center; + min-height: 40px; /* Adjust as needed for vertical centering */ +} + +.lds-ellipsis, +.lds-ellipsis div { + box-sizing: border-box; +} +.lds-ellipsis { + display: inline-block; + position: relative; + width: 24px; + height: 24px; +} +.lds-ellipsis div { + position: absolute; + top: 12px; + width: 6px; + height: 6px; + border-radius: 50%; + background: currentColor; + animation-timing-function: cubic-bezier(0, 1, 1, 0); +} +.lds-ellipsis div:nth-child(1) { + left: 2px; + animation: lds-ellipsis1 0.6s infinite; +} +.lds-ellipsis div:nth-child(2) { + left: 2px; + animation: lds-ellipsis2 0.6s infinite; +} +.lds-ellipsis div:nth-child(3) { + left: 12px; + animation: lds-ellipsis2 0.6s infinite; +} +.lds-ellipsis div:nth-child(4) { + left: 22px; + animation: lds-ellipsis3 0.6s infinite; +} +@keyframes lds-ellipsis1 { + 0% { + transform: scale(0); + } + 100% { + transform: scale(1); + } +} +@keyframes lds-ellipsis3 { + 0% { + transform: scale(1); + } + 100% { + transform: scale(0); + } +} +@keyframes lds-ellipsis2 { + 0% { + transform: translate(0, 0); + } + 100% { + transform: translate(24px, 0); + } +} + diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index 27b11b9..b89a468 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -175,27 +175,24 @@ async function checkGeoCoverage(geoValue) { } } -function getAvailableGeos(indicators) { - var availableGeos = null; - +async function getAvailableGeos(indicators) { const csrftoken = Cookies.get("csrftoken"); + const submitData = { indicators: indicators }; - const submitData = { - indicators: indicators + try { + const data = await $.ajax({ + url: "get_available_geos/", + type: "POST", + dataType: "json", + contentType: "application/json", + headers: { "X-CSRFToken": csrftoken }, + data: JSON.stringify(submitData), + }); + return data.geographic_granularities; + } catch (error) { + console.error("Error fetching available geos:", error); + return null; } - - $.ajax({ - url: "get_available_geos/", - type: "POST", - async: false, // Synchronous request to ensure availableGeos is populated before returning - dataType: "json", - contentType: "application/json", - headers: { "X-CSRFToken": csrftoken }, - data: JSON.stringify(submitData), - }).done(function (data) { - availableGeos = data.geographic_granularities; - }); - return availableGeos; } $("#geographic_value").on("select2:select", function (e) { @@ -208,20 +205,38 @@ $("#geographic_value").on("select2:select", function (e) { }); -$("#showSelectedIndicatorsButton").click(function () { +$("#showSelectedIndicatorsButton").click(async function () { alertPlaceholder.innerHTML = ""; - const availableGeos = getAvailableGeos(checkedIndicatorMembers); - const locationIds = $("#location_search").select2("data").map((item) => item.id); + + const prevSelectedIds = $('#geographic_value').val() || []; + if ($('#geographic_value').hasClass("select2-hidden-accessible")) { $('#geographic_value').select2('destroy').empty(); } + + // Show loader in #geographic_value + $('#geographic_value').hide(); + $('#geographic_value').after('
'); + + const availableGeos = await getAvailableGeos(checkedIndicatorMembers); + const locationIds = $("#location_search").select2("data").map((item) => item.id); + + // Remove loader and show Select2 + $('#geo-loader').remove(); + $('#geographic_value').show(); + $("#geographic_value").select2({ data: availableGeos, minimumInputLength: 0, maximumSelectionLength: 5, }); - $('#geographic_value').val(locationIds).trigger('change'); + + const availableGeoIds = availableGeos.flatMap(group => group.children.map(child => child.id)); + const preservedIds = prevSelectedIds.filter(id => availableGeoIds.includes(id)); + + const selectedGeos = [...locationIds, ...preservedIds]; + $('#geographic_value').val(selectedGeos).trigger('change'); if (!indicatorHandler.checkForCovidcastIndicators()) { $('#geographic_value').val(null).trigger('change'); $("#geographic_value").prop("disabled", true); diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 7fd4660..ffb60ff 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.1" +APP_VERSION = "1.0.2" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From 9e4a01293a1efaa4cd32bd4e74930a5e6f406b44 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 30 Jul 2025 12:23:50 +0300 Subject: [PATCH 09/49] Changed INILET choose location warning message --- src/assets/js/selectedIndicatorsModal.js | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index b89a468..04548d1 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -225,7 +225,7 @@ $("#showSelectedIndicatorsButton").click(async function () { // Remove loader and show Select2 $('#geo-loader').remove(); $('#geographic_value').show(); - + $("#geographic_value").select2({ data: availableGeos, minimumInputLength: 0, @@ -250,13 +250,9 @@ $("#showSelectedIndicatorsButton").click(async function () { } }) }); - nonCovidcastIndicatorSets = [...new Set(checkedIndicatorMembers.filter(indicator => indicator["_endpoint"] != "covidcast").map((indicator) => indicator["indicator_set"]))]; - var otherEndpointLocationsWarning = `` if (indicatorHandler.getFluviewIndicators().length > 0) { - $("#differentLocationNote").html(otherEndpointLocationsWarning) + var ilinetEndpointLocationsWarning = ''; + $("#differentLocationNote").html(ilinetEndpointLocationsWarning) if (document.getElementsByName("fluviewRegions").length === 0) { indicatorHandler.showFluviewRegions(); } else { From 3c66b514e3e5db477c01ed76a7dcc01ceb2469d4 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 30 Jul 2025 12:47:31 +0300 Subject: [PATCH 10/49] Changed cursor style when is pointed on location select dropdown arrow --- src/assets/css/style.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/assets/css/style.css b/src/assets/css/style.css index d600578..a825c07 100644 --- a/src/assets/css/style.css +++ b/src/assets/css/style.css @@ -1347,7 +1347,7 @@ h6 { top: 50%; right: 5px; transform: translateY(-50%); - pointer-events: none; + cursor: pointer; } .select2-container--default.select2-container--open .select2-selection--multiple:after { From b4348b99f331c557d940943582212b6f76d8f275 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 31 Jul 2025 19:25:25 +0300 Subject: [PATCH 11/49] Indicators that are not available for selected geos will not be plotted --- src/assets/js/indicatorHandler.js | 1 + src/assets/js/selectedIndicatorsModal.js | 14 ++++-- src/indicatorsets/views.py | 55 +++++++++++++----------- 3 files changed, 40 insertions(+), 30 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 79ba439..a2a38a7 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -211,6 +211,7 @@ class IndicatorHandler { fluviewRegions: fluviewRegions, apiKey: document.getElementById("apiKey").value, }; + console.log(this.indicators); const csrftoken = Cookies.get("csrftoken"); $.ajax({ url: "epivis/", diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index 04548d1..77ced08 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -133,11 +133,11 @@ function showNotCoveredGeoWarningMessage(notCoveredIndicators, geoValue) { var warningMessage = ""; notCoveredIndicators.forEach((indicator) => { if (currentMode === "epivis") { - warningMessage += `Indicator ${indicator.display_name} is not available for Location ${geoValue}
`; + warningMessage += `Indicator ${indicator.display_name} is not available for Location ${geoValue.text}
`; } else { var startDate = document.getElementById("start_date").value; var endDate = document.getElementById("end_date").value; - warningMessage += `Indicator ${indicator.display_name} is not available for Location ${geoValue} for the time period from ${startDate} to ${endDate}
`; + warningMessage += `Indicator ${indicator.display_name} is not available for Location ${geoValue.text} for the time period from ${startDate} to ${endDate}
`; } }); appendAlert(warningMessage, "warning"); @@ -164,6 +164,12 @@ async function checkGeoCoverage(geoValue) { e.signal === indicator.indicator ); if (!covered) { + if (!indicator["notCoveredGeos"]) { + indicator["notCoveredGeos"] = []; + } + if (!indicator["notCoveredGeos"].includes(geoValue)) { + indicator["notCoveredGeos"].push(geoValue); + } notCoveredIndicators.push(indicator); } }); @@ -199,7 +205,7 @@ $("#geographic_value").on("select2:select", function (e) { var geo = e.params.data; checkGeoCoverage(geo.id).then((notCoveredIndicators) => { if (notCoveredIndicators.length > 0) { - showNotCoveredGeoWarningMessage(notCoveredIndicators, geo.text); + showNotCoveredGeoWarningMessage(notCoveredIndicators, geo); } }); }); @@ -246,7 +252,7 @@ $("#showSelectedIndicatorsButton").click(async function () { $('#geographic_value').select2("data").forEach(geo => { checkGeoCoverage(geo.id).then((notCoveredIndicators) => { if (notCoveredIndicators.length > 0) { - showNotCoveredGeoWarningMessage(notCoveredIndicators, geo.text); + showNotCoveredGeoWarningMessage(notCoveredIndicators, geo); } }) }); diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index 7087882..576225f 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -234,28 +234,29 @@ def epivis(request): for indicator in indicators: if indicator["_endpoint"] == "covidcast": for geo in covidcast_geos: - geo_value = ( - geo["id"].split(":")[1].lower() - if geo["geoType"] in ["nation", "state"] - else geo["id"].split(":")[1] - ) - datasets.append( - { - "color": generate_random_color(), - "title": "value", - "params": { - "_endpoint": indicator["_endpoint"], - "data_source": indicator["data_source"], - "signal": indicator["indicator"], - "time_type": indicator["time_type"], - "geo_type": geo["geoType"], - "geo_value": geo_value, - "custom_title": generate_epivis_custom_title( - indicator, geo["text"] - ), - }, - } - ) + if geo["id"] not in indicator.get("notCoveredGeos", []): + geo_value = ( + geo["id"].split(":")[1].lower() + if geo["geoType"] in ["nation", "state"] + else geo["id"].split(":")[1] + ) + datasets.append( + { + "color": generate_random_color(), + "title": "value", + "params": { + "_endpoint": indicator["_endpoint"], + "data_source": indicator["data_source"], + "signal": indicator["indicator"], + "time_type": indicator["time_type"], + "geo_type": geo["geoType"], + "geo_value": geo_value, + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) elif indicator["_endpoint"] == "fluview": for geo in fluview_geos: datasets.append( @@ -273,10 +274,12 @@ def epivis(request): }, } ) - datasets_json = json.dumps({"datasets": datasets}) - datasets_b64 = base64.b64encode(datasets_json.encode("ascii")).decode("ascii") - response = {"epivis_url": f"{settings.EPIVIS_URL}#{datasets_b64}"} - return JsonResponse(response) + if datasets: + datasets_json = json.dumps({"datasets": datasets}) + datasets_b64 = base64.b64encode(datasets_json.encode("ascii")).decode("ascii") + return JsonResponse({"epivis_url": f"{settings.EPIVIS_URL}#{datasets_b64}"}) + else: + return JsonResponse({"epivis_url": settings.EPIVIS_URL}) def generate_export_data_url(request): From 7fdd8fde811b406cd45e4308432114ee0a035bbe Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 31 Jul 2025 19:25:48 +0300 Subject: [PATCH 12/49] Version bump --- src/epiportal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index ffb60ff..720394f 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.2" +APP_VERSION = "1.0.3" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From c42ebd74bebbf2a46c583ab18e32a756fdceb222 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 1 Aug 2025 10:07:36 +0300 Subject: [PATCH 13/49] Removed console.log statement --- src/assets/js/indicatorHandler.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index a2a38a7..79ba439 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -211,7 +211,6 @@ class IndicatorHandler { fluviewRegions: fluviewRegions, apiKey: document.getElementById("apiKey").value, }; - console.log(this.indicators); const csrftoken = Cookies.get("csrftoken"); $.ajax({ url: "epivis/", From 593ffad844775b97ea71b52522d5815715535f97 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 4 Aug 2025 16:20:47 +0300 Subject: [PATCH 14/49] Changed pill colors --- src/assets/css/custom_styles.css | 11 +++++++++++ src/templates/indicatorsets/indicatorSetsTable.html | 8 ++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/assets/css/custom_styles.css b/src/assets/css/custom_styles.css index 1466579..0da518f 100644 --- a/src/assets/css/custom_styles.css +++ b/src/assets/css/custom_styles.css @@ -247,3 +247,14 @@ tr:not(.odd-row) > .dtfc-fixed-end { } } + +.badge-pill-outline { + border: 1px solid #000; + background-color: #fff; + color: #000; + border-radius: 50rem; + padding: 0.35em 0.65em; + font-size: 0.85em; + margin-right: 0.4em; + margin-bottom: 0.25em; +} diff --git a/src/templates/indicatorsets/indicatorSetsTable.html b/src/templates/indicatorsets/indicatorSetsTable.html index 21fbe89..87feeea 100644 --- a/src/templates/indicatorsets/indicatorSetsTable.html +++ b/src/templates/indicatorsets/indicatorSetsTable.html @@ -272,20 +272,20 @@ @@ -295,7 +295,7 @@ From a6a8ae80fd88abc646e3f9a60fb8c4701aab6066 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 4 Aug 2025 16:21:50 +0300 Subject: [PATCH 15/49] Version bump --- src/epiportal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 720394f..ce38745 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.3" +APP_VERSION = "1.0.4" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From 63f3c775062737b982f93a19a8fb30aa82865f87 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 14 Aug 2025 16:45:14 +0300 Subject: [PATCH 16/49] Removed whitespace in the header --- src/templates/index.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/templates/index.html b/src/templates/index.html index 7da44d7..683223e 100644 --- a/src/templates/index.html +++ b/src/templates/index.html @@ -70,7 +70,7 @@ From 9d99bbde6c1dd45d9f306c6b5a17f2e8dbcd56de Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 9 Sep 2025 15:00:03 +0300 Subject: [PATCH 33/49] Added NIDSS Flu location selector --- src/assets/js/indicatorHandler.js | 45 +++++++++++++- src/assets/js/indicatorSetsTable.js | 2 +- src/assets/js/selectedIndicatorsModal.js | 74 +++++++++++++++++++----- src/indicatorsets/views.py | 3 + 4 files changed, 108 insertions(+), 16 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 79ba439..8aa2027 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -10,6 +10,7 @@ function dataLayerPush(payload) { class IndicatorHandler { constructor() { this.indicators = {}; + this.nonCovidcastIndicatorSets = []; } fluviewIndicatorsMapping = { @@ -125,6 +126,16 @@ class IndicatorHandler { { id: "jfk", text: "New York City" }, ]; + nidssFluLocations = [ + { id: 'nationwide', text: 'Taiwan National' }, + { id: 'central', text: 'Central' }, + { id: 'eastern', text: 'Eastern' }, + { id: 'kaoping', text: 'Kaoping' }, + { id: 'northern', text: 'Northern' }, + { id: 'southern', text: 'Southern' }, + { id: 'taipei', text: 'Taipei' }, + ]; + checkForCovidcastIndicators() { return this.indicators.some((indicator) => { return indicator["_endpoint"] === "covidcast"; @@ -151,6 +162,16 @@ class IndicatorHandler { return fluviewIndicators; } + getNIDSSFluIndicators() { + var nidssFluIndicators = []; + this.indicators.forEach((indicator) => { + if (indicator["_endpoint"] === "nidss_flu") { + nidssFluIndicators.push(indicator); + } + }); + return nidssFluIndicators; + } + getFromToDate(startDate, endDate, timeType) { if (timeType === "week") { $.ajax({ @@ -182,7 +203,7 @@ class IndicatorHandler { showFluviewRegions() { var fluviewRegionSelect = ` -
+
@@ -201,6 +222,27 @@ class IndicatorHandler { } } + showNIDSSFluLocations() { + var nidssFluRegionSelect = ` +
+
+ +
+
+ +
+
`; + if ($("#otherEndpointLocations").length) { + $("#otherEndpointLocations").append(nidssFluRegionSelect); + $("#nidssFluRegions").select2({ + placeholder: "Select Taiwanese ILI Location(s)", + data: this.nidssFluLocations, + allowClear: true, + width: "100%", + }); + } + } + plotData() { const covidCastGeographicValues = $("#geographic_value").select2("data"); @@ -227,6 +269,7 @@ class IndicatorHandler { indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewRegions"]), + epivisUrl: data["epivis_url"], apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } diff --git a/src/assets/js/indicatorSetsTable.js b/src/assets/js/indicatorSetsTable.js index 40d47ec..ee81152 100644 --- a/src/assets/js/indicatorSetsTable.js +++ b/src/assets/js/indicatorSetsTable.js @@ -83,7 +83,7 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) { ).length; var checkboxTitle = ""; checked = checked ? "checked" : ""; - disabled = indicator.endpoint !== "covidcast" && indicator.endpoint !== "fluview" ? "disabled" : ""; + disabled = indicator.endpoint !== "covidcast" && indicator.endpoint !== "fluview" && indicator.endpoint !== "nidss_flu" ? "disabled" : ""; sourceType = indicator.source_type; var restricted = indicator.restricted != "No"; if (disabled === "disabled") { diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index 77ced08..bbc914c 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -40,6 +40,11 @@ function addSelectedIndicator(element) { element.dataset.indicatorSet, element.dataset.indicator ); + if (element.dataset.endpoint !== "covidcast" && !indicatorHandler.nonCovidcastIndicatorSets.includes(element.dataset.indicatorSet)) { + indicatorHandler.nonCovidcastIndicatorSets.push( + element.dataset.indicatorSet + ); + } } else { checkedIndicatorMembers = checkedIndicatorMembers.filter( (indicator) => indicator.indicator !== element.dataset.indicator @@ -49,6 +54,15 @@ function addSelectedIndicator(element) { `${element.dataset.datasource}_${element.dataset.indicator}` ) .remove(); + const indicatorSet = element.dataset.indicatorSet; + const stillExist = checkedIndicatorMembers.some( + (indicator) => indicator.indicator_set === indicatorSet + ); + if (!stillExist && indicatorHandler.nonCovidcastIndicatorSets.includes(indicatorSet)) { + indicatorHandler.nonCovidcastIndicatorSets = indicatorHandler.nonCovidcastIndicatorSets.filter( + (set) => set !== indicatorSet + ); + } } indicatorHandler.indicators = checkedIndicatorMembers; @@ -210,8 +224,54 @@ $("#geographic_value").on("select2:select", function (e) { }); }); +function showFluviewLocationSelect() { + if (indicatorHandler.getFluviewIndicators().length > 0) { + if (document.getElementsByName("fluviewRegions").length === 0) { + indicatorHandler.showFluviewRegions(); + } else { + // IF code goes here, we assume that otherEndpointLocationWarning & fluviewRegion selector is already on the page, but is just hidden, so we should just show it. + $("#fluviewDiv").show(); + } + } else { + // If there are no non-covidcast indicators selected (only fluview is supported for now) then hide otherEndpointLocationWarning & fliviewRegions selector. + $("#fluviewRegions").val(null).trigger("change"); + $("#fluviewDiv").hide(); + } +} + +function showNIDSSFluLocationSelect() { + if (indicatorHandler.getNIDSSFluIndicators().length > 0) { + if (document.getElementsByName("nidssFluRegions").length === 0) { + indicatorHandler.showNIDSSFluLocations(); + } else { + // IF code goes here, we assume that otherEndpointLocationWarning & nidssRegion selector is already on the page, but is just hidden, so we should just show it. + $("#nidssFluDiv").show(); + } + } else { + // If there are no non-covidcast indicators selected (only fluview is supported for now) then hide otherEndpointLocationWarning & fliviewRegions selector. + $("#nidssFluRegions").val(null).trigger("change"); + $("#nidssFluDiv").hide(); + } +} + +function showNonDelphiIndicatorSetsLocations() { + if (indicatorHandler.nonCovidcastIndicatorSets.length > 0) { + + var otherEndpointIndicatorSetsLocationMessage = `` + $("#differentLocationNote").html(otherEndpointIndicatorSetsLocationMessage); + showFluviewLocationSelect(); + showNIDSSFluLocationSelect(); + $("#otherEndpointLocationsWrapper").show(); + } else { + $("#differentLocationNote").html(""); + $("#otherEndpointLocationsWrapper").hide(); + } +} + + $("#showSelectedIndicatorsButton").click(async function () { + showNonDelphiIndicatorSetsLocations(); alertPlaceholder.innerHTML = ""; const prevSelectedIds = $('#geographic_value').val() || []; @@ -256,20 +316,6 @@ $("#showSelectedIndicatorsButton").click(async function () { } }) }); - if (indicatorHandler.getFluviewIndicators().length > 0) { - var ilinetEndpointLocationsWarning = ''; - $("#differentLocationNote").html(ilinetEndpointLocationsWarning) - if (document.getElementsByName("fluviewRegions").length === 0) { - indicatorHandler.showFluviewRegions(); - } else { - // IF code goes here, we assume that otherEndpointLocationWarning & fluviewRegion selector is already on the page, but is just hidden, so we should just show it. - $("#otherEndpointLocationsWrapper").show(); - } - } else { - // If there are no non-covidcast indicators selected (only fluview is supported for now) then hide otherEndpointLocationWarning & fliviewRegions selector. - $("#fluviewRegions").val(null).trigger("change"); - $("#otherEndpointLocationsWrapper").hide(); - } }); diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index 44d0108..d2353b1 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -307,6 +307,9 @@ def epivis(request): geo_value=geo["id"], api_key=api_key, ) # noqa: E501 + + elif indicator["_endpoint"] == "nidss_flu": + pass if datasets: datasets_json = json.dumps({"datasets": datasets}) datasets_b64 = base64.b64encode(datasets_json.encode("ascii")).decode( From 6743f0e8d65be0f13e91d5010bdcb517cbbe086a Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 9 Sep 2025 15:19:24 +0300 Subject: [PATCH 34/49] Aligned fluviewLocations namings --- src/assets/js/indicatorHandler.js | 42 ++++++++++++------------ src/assets/js/selectedIndicatorsModal.js | 6 ++-- src/indicatorsets/views.py | 8 ++--- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 8aa2027..36c5ea3 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -43,7 +43,7 @@ class IndicatorHandler { { value: "UT", label: "UT" }, ]; - fluviewRegions = [ + fluviewLocations = [ { id: "nat", text: "U.S. National" }, { id: "hhs1", text: "HHS Region 1" }, { id: "hhs2", text: "HHS Region 2" }, @@ -201,21 +201,21 @@ class IndicatorHandler { return request; } - showFluviewRegions() { - var fluviewRegionSelect = ` + showfluviewLocations() { + var fluviewLocationselect = `
- +
- +
`; if ($("#otherEndpointLocations").length) { - $("#otherEndpointLocations").append(fluviewRegionSelect); - $("#fluviewRegions").select2({ + $("#otherEndpointLocations").append(fluviewLocationselect); + $("#fluviewLocations").select2({ placeholder: "Select ILINet Location(s)", - data: this.fluviewRegions, + data: this.fluviewLocations, allowClear: true, width: "100%", }); @@ -246,11 +246,11 @@ class IndicatorHandler { plotData() { const covidCastGeographicValues = $("#geographic_value").select2("data"); - const fluviewRegions = $("#fluviewRegions").select2("data"); + const fluviewLocations = $("#fluviewLocations").select2("data"); const submitData = { indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, - fluviewRegions: fluviewRegions, + fluviewLocations: fluviewLocations, apiKey: document.getElementById("apiKey").value, }; const csrftoken = Cookies.get("csrftoken"); @@ -268,8 +268,8 @@ class IndicatorHandler { formMode: "epivis", indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), - fluviewGeoValues: JSON.stringify(submitData["fluviewRegions"]), - + fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), + epivisUrl: data["epivis_url"], apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } @@ -279,7 +279,7 @@ class IndicatorHandler { } exportData() { - var fluviewRegions = $("#fluviewRegions").select2("data"); + var fluviewLocations = $("#fluviewLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -290,7 +290,7 @@ class IndicatorHandler { end_date: document.getElementById("end_date").value, indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, - fluviewRegions: fluviewRegions, + fluviewLocations: fluviewLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -310,7 +310,7 @@ class IndicatorHandler { formEndDate: submitData["end_date"], indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), - fluviewGeoValues: JSON.stringify(submitData["fluviewRegions"]), + fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -320,7 +320,7 @@ class IndicatorHandler { previewData() { $('#loader').show(); - var fluviewRegions = $("#fluviewRegions").select2("data"); + var fluviewLocations = $("#fluviewLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -331,7 +331,7 @@ class IndicatorHandler { end_date: document.getElementById("end_date").value, indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, - fluviewRegions: fluviewRegions, + fluviewLocations: fluviewLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -350,7 +350,7 @@ class IndicatorHandler { formEndDate: submitData["end_date"], indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), - fluviewGeoValues: JSON.stringify(submitData["fluviewRegions"]), + fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -361,7 +361,7 @@ class IndicatorHandler { createQueryCode() { - var fluviewRegions = $("#fluviewRegions").select2("data"); + var fluviewLocations = $("#fluviewLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -373,7 +373,7 @@ class IndicatorHandler { end_date: document.getElementById("end_date").value, indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, - fluviewRegions: fluviewRegions, + fluviewLocations: fluviewLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -400,7 +400,7 @@ class IndicatorHandler { formEndDate: submitData["end_date"], indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), - fluviewGeoValues: JSON.stringify(submitData["fluviewRegions"]), + fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index bbc914c..f79632f 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -226,15 +226,15 @@ $("#geographic_value").on("select2:select", function (e) { function showFluviewLocationSelect() { if (indicatorHandler.getFluviewIndicators().length > 0) { - if (document.getElementsByName("fluviewRegions").length === 0) { - indicatorHandler.showFluviewRegions(); + if (document.getElementsByName("fluviewLocations").length === 0) { + indicatorHandler.showfluviewLocations(); } else { // IF code goes here, we assume that otherEndpointLocationWarning & fluviewRegion selector is already on the page, but is just hidden, so we should just show it. $("#fluviewDiv").show(); } } else { // If there are no non-covidcast indicators selected (only fluview is supported for now) then hide otherEndpointLocationWarning & fliviewRegions selector. - $("#fluviewRegions").val(null).trigger("change"); + $("#fluviewLocations").val(null).trigger("change"); $("#fluviewDiv").hide(); } } diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index d2353b1..a9a29cf 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -234,7 +234,7 @@ def epivis(request): data = json.loads(request.body) indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", []) - fluview_geos = data.get("fluviewRegions", []) + fluview_geos = data.get("fluviewLocations", []) api_key = data.get("apiKey", "") form_activity_logger.info( mode="epivis", @@ -329,7 +329,7 @@ def generate_export_data_url(request): end_date = data.get("end_date", "") indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", {}) - fluview_geos = data.get("fluviewRegions", []) + fluview_geos = data.get("fluviewLocations", []) api_key = data.get("apiKey", None) form_activity_logger.info( mode="data_export", @@ -398,7 +398,7 @@ def preview_data(request): end_date = data.get("end_date", "") indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", {}) - fluview_geos = data.get("fluviewRegions", []) + fluview_geos = data.get("fluviewLocations", []) api_key = data.get("apiKey", None) preview_data = [] @@ -507,7 +507,7 @@ def create_query_code(request): end_date = data.get("end_date", "") indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", {}) - fluview_geos = data.get("fluviewRegions", []) + fluview_geos = data.get("fluviewLocations", []) api_key = data.get("apiKey", None) python_code_blocks = [ dedent( From e4e293781e934a0e2cd816c208315aec51a428b4 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 9 Sep 2025 22:47:55 +0300 Subject: [PATCH 35/49] Added Taiwaneese ILI support --- src/assets/js/indicatorHandler.js | 23 +++-- src/assets/js/selectedIndicatorsModal.js | 8 +- src/indicatorsets/views.py | 106 ++++++++++++++++++++++- 3 files changed, 126 insertions(+), 11 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 36c5ea3..ab8201f 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -223,18 +223,18 @@ class IndicatorHandler { } showNIDSSFluLocations() { - var nidssFluRegionSelect = ` + var nidssFluLocationselect = `
- +
- +
`; if ($("#otherEndpointLocations").length) { - $("#otherEndpointLocations").append(nidssFluRegionSelect); - $("#nidssFluRegions").select2({ + $("#otherEndpointLocations").append(nidssFluLocationselect); + $("#nidssFluLocations").select2({ placeholder: "Select Taiwanese ILI Location(s)", data: this.nidssFluLocations, allowClear: true, @@ -247,10 +247,12 @@ class IndicatorHandler { const covidCastGeographicValues = $("#geographic_value").select2("data"); const fluviewLocations = $("#fluviewLocations").select2("data"); + const nidssFluLocations = $("#nidssFluLocations").select2("data"); const submitData = { indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, + nidssFluLocations: nidssFluLocations, apiKey: document.getElementById("apiKey").value, }; const csrftoken = Cookies.get("csrftoken"); @@ -269,7 +271,7 @@ class IndicatorHandler { indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), - + nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), epivisUrl: data["epivis_url"], apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } @@ -280,6 +282,7 @@ class IndicatorHandler { exportData() { var fluviewLocations = $("#fluviewLocations").select2("data"); + var nidssFluLocations = $("#nidssFluLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -291,6 +294,7 @@ class IndicatorHandler { indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, + nidssFluLocations: nidssFluLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -311,6 +315,7 @@ class IndicatorHandler { indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), + nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -321,6 +326,7 @@ class IndicatorHandler { previewData() { $('#loader').show(); var fluviewLocations = $("#fluviewLocations").select2("data"); + var nidssFluLocations = $("#nidssFluLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -332,6 +338,7 @@ class IndicatorHandler { indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, + nidssFluLocations: nidssFluLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -351,6 +358,7 @@ class IndicatorHandler { indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), + nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -362,6 +370,7 @@ class IndicatorHandler { createQueryCode() { var fluviewLocations = $("#fluviewLocations").select2("data"); + var nidssFluLocations = $("#nidssFluLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -374,6 +383,7 @@ class IndicatorHandler { indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, + nidssFluLocations: nidssFluLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -401,6 +411,7 @@ class IndicatorHandler { indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), + nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index f79632f..9c9133c 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -233,7 +233,7 @@ function showFluviewLocationSelect() { $("#fluviewDiv").show(); } } else { - // If there are no non-covidcast indicators selected (only fluview is supported for now) then hide otherEndpointLocationWarning & fliviewRegions selector. + // If there are no non-covidcast indicators selected then hide otherEndpointLocationWarning & fluviewLocations selector. $("#fluviewLocations").val(null).trigger("change"); $("#fluviewDiv").hide(); } @@ -241,15 +241,15 @@ function showFluviewLocationSelect() { function showNIDSSFluLocationSelect() { if (indicatorHandler.getNIDSSFluIndicators().length > 0) { - if (document.getElementsByName("nidssFluRegions").length === 0) { + if (document.getElementsByName("nidssFluLocations").length === 0) { indicatorHandler.showNIDSSFluLocations(); } else { // IF code goes here, we assume that otherEndpointLocationWarning & nidssRegion selector is already on the page, but is just hidden, so we should just show it. $("#nidssFluDiv").show(); } } else { - // If there are no non-covidcast indicators selected (only fluview is supported for now) then hide otherEndpointLocationWarning & fliviewRegions selector. - $("#nidssFluRegions").val(null).trigger("change"); + // If there are no non-covidcast indicators selected then hide otherEndpointLocationWarning & nidssFluLocations selector. + $("#nidssFluLocations").val(null).trigger("change"); $("#nidssFluDiv").hide(); } } diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index a9a29cf..4d307e7 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -235,6 +235,7 @@ def epivis(request): indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", []) fluview_geos = data.get("fluviewLocations", []) + nidss_flu_locations = data.get("nidssFluLocations", []) api_key = data.get("apiKey", "") form_activity_logger.info( mode="epivis", @@ -309,7 +310,28 @@ def epivis(request): ) # noqa: E501 elif indicator["_endpoint"] == "nidss_flu": - pass + for geo in nidss_flu_locations: + datasets.append( + { + "color": generate_random_color(), + "title": indicator["indicator"], + "params": { + "_endpoint": indicator["_endpoint"], + "regions": geo["id"], + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) + form_activity_logger.info( + mode="epivis", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_value=geo["id"], + api_key=api_key, + ) if datasets: datasets_json = json.dumps({"datasets": datasets}) datasets_b64 = base64.b64encode(datasets_json.encode("ascii")).decode( @@ -330,6 +352,7 @@ def generate_export_data_url(request): indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", {}) fluview_geos = data.get("fluviewLocations", []) + nidss_flu_locations = data.get("nidssFluLocations", []) api_key = data.get("apiKey", None) form_activity_logger.info( mode="data_export", @@ -383,6 +406,22 @@ def generate_export_data_url(request): data_export_commands.append( f'wget --content-disposition {data_export_url}' ) + if nidss_flu_locations: + regions = ",".join([region["id"] for region in nidss_flu_locations]) + date_from, date_to = get_epiweek(start_date, end_date) + form_activity_logger.info( + mode="data_export", + endpoint="nidss_flu", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + data_export_url = f"{settings.EPIDATA_URL}nidss_flu/?regions={regions}&epiweeks={date_from}-{date_to}&format=csv" + if api_key: + data_export_url += f"&api_key={api_key}" + data_export_commands.append( + f'wget --content-disposition {data_export_url}' + ) data_export_block = data_export_block.format("
".join(data_export_commands)) response = { "data_export_block": data_export_block, @@ -399,6 +438,7 @@ def preview_data(request): indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", {}) fluview_geos = data.get("fluviewLocations", []) + nidss_flu_locations = data.get("nidssFluLocations", []) api_key = data.get("apiKey", None) preview_data = [] @@ -497,6 +537,39 @@ def preview_data(request): "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", } return JsonResponse(preview_data, safe=False) + if nidss_flu_locations: + regions = ",".join([region["id"] for region in nidss_flu_locations]) + date_from, date_to = get_epiweek(start_date, end_date) + params = { + "regions": regions, + "epiweeks": f"{date_from}-{date_to}", + "api_key": api_key if api_key else settings.EPIDATA_API_KEY, + } + form_activity_logger.info( + mode="data_export", + endpoint="nidss_flu", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + response = requests.get(f"{settings.EPIDATA_URL}nidss_flu", params=params) + if response.status_code == 200: + data = response.json() + if len(data["epidata"]): + preview_data.append( + { + "epidata": data["epidata"][0], + "result": data["result"], + "message": data["message"], + } + ) + elif response.status_code == 401: + preview_data = { + "epidata": [], + "result": -2, + "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", + } + return JsonResponse(preview_data, safe=False) return JsonResponse(preview_data, safe=False) @@ -508,6 +581,8 @@ def create_query_code(request): indicators = data.get("indicators", []) covidcast_geos = data.get("covidCastGeographicValues", {}) fluview_geos = data.get("fluviewLocations", []) + nidss_flu_locations = data.get("nidssFluLocations", []) + print(nidss_flu_locations) api_key = data.get("apiKey", None) python_code_blocks = [ dedent( @@ -637,6 +712,35 @@ def create_query_code(request): """ ) r_code_blocks.append(r_code_block) + if nidss_flu_locations: + print(nidss_flu_locations) + regions = ",".join([region["id"] for region in nidss_flu_locations]) + start_week, end_week = get_epiweek(start_date, end_date) + python_code_block = dedent( + f"""\ + nidss_flu_df = epidata.pub_nidss_flu( + regions="{regions}", + epiweeks="{start_week}-{end_week}", + ).df() + """ + ) + form_activity_logger.info( + mode="data_export", + endpoint="nidss_flu", + regions=regions, + epiweeks=f"{start_week}-{end_week}", + api_key=api_key, + ) # noqa: E501 + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_{data_source.replace("-", "_")} <- pub_nidss_flu( + regions = "{regions}", + epiweeks = epirange({start_week}, {end_week}) + ) + """ + ) + r_code_blocks.append(r_code_block) python_code_blocks.append("") r_code_blocks.append("") return JsonResponse( From 7874e4ef9c78df23da92fa22913f5930d261b13c Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 10 Sep 2025 16:27:00 +0300 Subject: [PATCH 36/49] Removed print statements --- src/indicatorsets/views.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index 4d307e7..ae2c76c 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -582,7 +582,6 @@ def create_query_code(request): covidcast_geos = data.get("covidCastGeographicValues", {}) fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) - print(nidss_flu_locations) api_key = data.get("apiKey", None) python_code_blocks = [ dedent( @@ -713,7 +712,6 @@ def create_query_code(request): ) r_code_blocks.append(r_code_block) if nidss_flu_locations: - print(nidss_flu_locations) regions = ",".join([region["id"] for region in nidss_flu_locations]) start_week, end_week = get_epiweek(start_date, end_date) python_code_block = dedent( From a107de75002609f58f81e852665341e48013cc0b Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Wed, 10 Sep 2025 16:45:30 +0300 Subject: [PATCH 37/49] Version bump --- src/epiportal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 84b5146..90f1377 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.5" +APP_VERSION = "1.0.6" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From 323493b7d8241b13df3311a76dd06c711415d0fd Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 12 Sep 2025 16:42:06 +0300 Subject: [PATCH 38/49] Added NIDSS Dengue Indicator Set support --- src/assets/js/indicatorHandler.js | 121 ++++++++++++++++++----- src/assets/js/indicatorSetsTable.js | 2 +- src/assets/js/selectedIndicatorsModal.js | 16 +++ src/epiportal/settings.py | 2 +- src/indicatorsets/views.py | 103 +++++++++++++++++++ 5 files changed, 219 insertions(+), 25 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index ab8201f..80cb8a0 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -19,28 +19,28 @@ class IndicatorHandler { }; fluSurvRegions = [ - { value: "network_all", label: "Entire Network" }, - { value: "network_eip", label: "EIP Netowrk" }, - { value: "network_ihsp", label: "IHSP Network" }, - { value: "CA", label: "CA" }, - { value: "CO", label: "CO" }, - { value: "CT", label: "CT" }, - { value: "GA", label: "GA" }, - { value: "IA", label: "IA" }, - { value: "ID", label: "ID" }, - { value: "MD", label: "MD" }, - { value: "MI", label: "MI" }, - { value: "MN", label: "MN" }, - { value: "NM", label: "NM" }, - { value: "NY_albany", label: "NY (Albany)" }, - { value: "NY_rochester", label: "NY (Rochester)" }, - { value: "OH", label: "OH" }, - { value: "OK", label: "OK" }, - { value: "OR", label: "OR" }, - { value: "RI", label: "RI" }, - { value: "SD", label: "SD" }, - { value: "TN", label: "TN" }, - { value: "UT", label: "UT" }, + { id: "network_all", text: "Entire Network" }, + { id: "network_eip", text: "EIP Netowrk" }, + { id: "network_ihsp", text: "IHSP Network" }, + { id: "CA", text: "CA" }, + { id: "CO", text: "CO" }, + { id: "CT", text: "CT" }, + { id: "GA", text: "GA" }, + { id: "IA", text: "IA" }, + { id: "ID", text: "ID" }, + { id: "MD", text: "MD" }, + { id: "MI", text: "MI" }, + { id: "MN", text: "MN" }, + { id: "NM", text: "NM" }, + { id: "NY_albany", text: "NY (Albany)" }, + { id: "NY_rochester", text: "NY (Rochester)" }, + { id: "OH", text: "OH" }, + { id: "OK", text: "OK" }, + { id: "OR", text: "OR" }, + { id: "RI", text: "RI" }, + { id: "SD", text: "SD" }, + { id: "TN", text: "TN" }, + { id: "UT", text: "UT" }, ]; fluviewLocations = [ @@ -136,6 +136,38 @@ class IndicatorHandler { { id: 'taipei', text: 'Taipei' }, ]; + nidssDengueLocations = [ + { id: 'nationwide', text: 'Taiwan National' }, + { id: 'central', text: 'Central' }, + { id: 'eastern', text: 'Eastern' }, + { id: 'kaoping', text: 'Kaoping' }, + { id: 'northern', text: 'Northern' }, + { id: 'southern', text: 'Southern' }, + { id: 'taipei', text: 'Taipei' }, + { id: 'changhua_county', text: 'Changhua County' }, + { id: 'chiayi_city', text: 'Chiayi City' }, + { id: 'chiayi_county', text: 'Chiayi County' }, + { id: 'hsinchu_city', text: 'Hsinchu City' }, + { id: 'hsinchu_county', text: 'Hsinchu County' }, + { id: 'hualien_county', text: 'Hualien County' }, + { id: 'kaohsiung_city', text: 'Kaohsiung City' }, + { id: 'keelung_city', text: 'Keelung City' }, + { id: 'kinmen_county', text: 'Kinmen County' }, + { id: 'lienchiang_county', text: 'Lienchiang County' }, + { id: 'miaoli_county', text: 'Miaoli County' }, + { id: 'nantou_county', text: 'Nantou County' }, + { id: 'new_taipei_city', text: 'New taipei City' }, + { id: 'penghu_county', text: 'Penghu County' }, + { id: 'pingtung_county', text: 'Pingtung County' }, + { id: 'taichung_city', text: 'Taichung City' }, + { id: 'tainan_city', text: 'Tainan City' }, + { id: 'taipei_city', text: 'Taipei City' }, + { id: 'taitung_county', text: 'Taitung County' }, + { id: 'taoyuan_city', text: 'Taoyuan City' }, + { id: 'yilan_county', text: 'Yilan County' }, + { id: 'yunlin_county', text: 'Yunlin County' }, + ]; + checkForCovidcastIndicators() { return this.indicators.some((indicator) => { return indicator["_endpoint"] === "covidcast"; @@ -172,6 +204,16 @@ class IndicatorHandler { return nidssFluIndicators; } + getNIDSSDengueIndicators() { + var nidssDengueIndicators = []; + this.indicators.forEach((indicator) => { + if (indicator["_endpoint"] === "nidss_dengue") { + nidssDengueIndicators.push(indicator); + } + }); + return nidssDengueIndicators; + } + getFromToDate(startDate, endDate, timeType) { if (timeType === "week") { $.ajax({ @@ -243,16 +285,39 @@ class IndicatorHandler { } } + showNIDSSDengueLocations() { + var nidssDengueLocationselect = ` +
+
+ +
+
+ +
+
`; + if ($("#otherEndpointLocations").length) { + $("#otherEndpointLocations").append(nidssDengueLocationselect); + $("#nidssDengueLocations").select2({ + placeholder: "Select Taiwanese Dengue Cases Location(s)", + data: this.nidssDengueLocations, + allowClear: true, + width: "100%", + }); + } + } + plotData() { const covidCastGeographicValues = $("#geographic_value").select2("data"); const fluviewLocations = $("#fluviewLocations").select2("data"); const nidssFluLocations = $("#nidssFluLocations").select2("data"); + const nidssDengueLocations = $("#nidssDengueLocations").select2("data"); const submitData = { indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, + nidssDengueLocations: nidssDengueLocations, apiKey: document.getElementById("apiKey").value, }; const csrftoken = Cookies.get("csrftoken"); @@ -272,6 +337,7 @@ class IndicatorHandler { covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), + nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), epivisUrl: data["epivis_url"], apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } @@ -283,6 +349,7 @@ class IndicatorHandler { exportData() { var fluviewLocations = $("#fluviewLocations").select2("data"); var nidssFluLocations = $("#nidssFluLocations").select2("data"); + var nidssDengueLocations = $("#nidssDengueLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -295,6 +362,7 @@ class IndicatorHandler { covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, + nidssDengueLocations: nidssDengueLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -316,6 +384,7 @@ class IndicatorHandler { covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), + nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -327,6 +396,7 @@ class IndicatorHandler { $('#loader').show(); var fluviewLocations = $("#fluviewLocations").select2("data"); var nidssFluLocations = $("#nidssFluLocations").select2("data"); + var nidssDengueLocations = $("#nidssDengueLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -339,6 +409,7 @@ class IndicatorHandler { covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, + nidssDengueLocations: nidssDengueLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -359,6 +430,7 @@ class IndicatorHandler { covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), + nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -371,6 +443,7 @@ class IndicatorHandler { var fluviewLocations = $("#fluviewLocations").select2("data"); var nidssFluLocations = $("#nidssFluLocations").select2("data"); + var nidssDengueLocations = $("#nidssDengueLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -384,6 +457,7 @@ class IndicatorHandler { covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, + nidssDengueLocations: nidssDengueLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -412,12 +486,13 @@ class IndicatorHandler { covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), + nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); createQueryCodePython += data["python_code_blocks"].join("
"); createQueryCodeR += data["r_code_blocks"].join("
"); - $('#modeSubmitResult').html(createQueryCodePython+"
"+createQueryCodeR); + $('#modeSubmitResult').html(createQueryCodePython + "
" + createQueryCodeR); }); } diff --git a/src/assets/js/indicatorSetsTable.js b/src/assets/js/indicatorSetsTable.js index ee81152..26ec369 100644 --- a/src/assets/js/indicatorSetsTable.js +++ b/src/assets/js/indicatorSetsTable.js @@ -83,7 +83,7 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) { ).length; var checkboxTitle = ""; checked = checked ? "checked" : ""; - disabled = indicator.endpoint !== "covidcast" && indicator.endpoint !== "fluview" && indicator.endpoint !== "nidss_flu" ? "disabled" : ""; + disabled = indicator.endpoint !== "covidcast" && indicator.endpoint !== "fluview" && indicator.endpoint !== "nidss_flu" && indicator.endpoint !== "nidss_dengue" ? "disabled" : ""; sourceType = indicator.source_type; var restricted = indicator.restricted != "No"; if (disabled === "disabled") { diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index 9c9133c..d81c61e 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -254,6 +254,21 @@ function showNIDSSFluLocationSelect() { } } +function showNIDSSDengueLocationSelect() { + if (indicatorHandler.getNIDSSDengueIndicators().length > 0) { + if (document.getElementsByName("nidssDengueLocations").length === 0) { + indicatorHandler.showNIDSSDengueLocations(); + } else { + // IF code goes here, we assume that otherEndpointLocationWarning & nidssRegion selector is already on the page, but is just hidden, so we should just show it. + $("#nidssDengueDiv").show(); + } + } else { + // If there are no non-covidcast indicators selected then hide otherEndpointLocationWarning & nidssDengueLocations selector. + $("#nidssDengueLocations").val(null).trigger("change"); + $("#nidssDengueDiv").hide(); + } +} + function showNonDelphiIndicatorSetsLocations() { if (indicatorHandler.nonCovidcastIndicatorSets.length > 0) { @@ -261,6 +276,7 @@ function showNonDelphiIndicatorSetsLocations() { $("#differentLocationNote").html(otherEndpointIndicatorSetsLocationMessage); showFluviewLocationSelect(); showNIDSSFluLocationSelect(); + showNIDSSDengueLocationSelect(); $("#otherEndpointLocationsWrapper").show(); } else { $("#differentLocationNote").html(""); diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 90f1377..c7c0359 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.6" +APP_VERSION = "1.0.7" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index ae2c76c..b14da09 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -236,6 +236,7 @@ def epivis(request): covidcast_geos = data.get("covidCastGeographicValues", []) fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) + nidss_dengue_locations = data.get("nidssDengueLocations", []) api_key = data.get("apiKey", "") form_activity_logger.info( mode="epivis", @@ -332,6 +333,29 @@ def epivis(request): geo_value=geo["id"], api_key=api_key, ) + elif indicator["_endpoint"] == "nidss_dengue": + for geo in nidss_dengue_locations: + datasets.append( + { + "color": generate_random_color(), + "title": indicator["indicator"], + "params": { + "_endpoint": indicator["_endpoint"], + "locations": geo["id"], + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) + form_activity_logger.info( + mode="epivis", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_value=geo["id"], + api_key=api_key, + ) if datasets: datasets_json = json.dumps({"datasets": datasets}) datasets_b64 = base64.b64encode(datasets_json.encode("ascii")).decode( @@ -353,6 +377,7 @@ def generate_export_data_url(request): covidcast_geos = data.get("covidCastGeographicValues", {}) fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) + nidss_dengue_locations = data.get("nidssDengueLocations", []) api_key = data.get("apiKey", None) form_activity_logger.info( mode="data_export", @@ -422,6 +447,22 @@ def generate_export_data_url(request): data_export_commands.append( f'wget --content-disposition {data_export_url}' ) + if nidss_dengue_locations: + regions = ",".join([region["id"] for region in nidss_dengue_locations]) + date_from, date_to = get_epiweek(start_date, end_date) + form_activity_logger.info( + mode="data_export", + endpoint="nidss_dengue", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + data_export_url = f"{settings.EPIDATA_URL}nidss_dengue/?locations={regions}&epiweeks={date_from}-{date_to}&format=csv" # fmt: skip + if api_key: + data_export_url += f"&api_key={api_key}" + data_export_commands.append( + f'wget --content-disposition {data_export_url}' + ) data_export_block = data_export_block.format("
".join(data_export_commands)) response = { "data_export_block": data_export_block, @@ -439,6 +480,7 @@ def preview_data(request): covidcast_geos = data.get("covidCastGeographicValues", {}) fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) + nidss_dengue_locations = data.get("nidssDengueLocations", []) api_key = data.get("apiKey", None) preview_data = [] @@ -570,6 +612,38 @@ def preview_data(request): "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", } return JsonResponse(preview_data, safe=False) + if nidss_dengue_locations: + regions = ",".join([region["id"] for region in nidss_dengue_locations]) + date_from, date_to = get_epiweek(start_date, end_date) + params = { + "locations": regions, + "epiweeks": f"{date_from}-{date_to}", + "api_key": api_key if api_key else settings.EPIDATA_API_KEY, + } + form_activity_logger.info( + mode="data_export", + endpoint="nidss_dengue", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + response = requests.get(f"{settings.EPIDATA_URL}nidss_dengue", params=params) + if response.status_code == 200: + data = response.json() + if len(data["epidata"]): + preview_data.append( + { + "epidata": data["epidata"][0], + "result": data["result"], + "message": data["message"], + } + ) + elif response.status_code == 401: + preview_data = { + "epidata": [], + "result": -2, + "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", + } return JsonResponse(preview_data, safe=False) @@ -582,6 +656,7 @@ def create_query_code(request): covidcast_geos = data.get("covidCastGeographicValues", {}) fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) + nidss_dengue_locations = data.get("nidssDengueLocations", []) api_key = data.get("apiKey", None) python_code_blocks = [ dedent( @@ -739,6 +814,34 @@ def create_query_code(request): """ ) r_code_blocks.append(r_code_block) + if nidss_dengue_locations: + regions = ",".join([region["id"] for region in nidss_dengue_locations]) + start_week, end_week = get_epiweek(start_date, end_date) + python_code_block = dedent( + f"""\ + nidss_dengue_df = epidata.pub_nidss_dengue( + locations="{regions}", + epiweeks="{start_week}-{end_week}", + ).df() + """ + ) + form_activity_logger.info( + mode="data_export", + endpoint="nidss_dengue", + regions=regions, + epiweeks=f"{start_week}-{end_week}", + api_key=api_key, + ) # noqa: E501 + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_{data_source.replace("-", "_")} <- pub_nidss_dengue( + locations = "{regions}", + epiweeks = epirange({start_week}, {end_week}) + ) + """ + ) + r_code_blocks.append(r_code_block) python_code_blocks.append("") r_code_blocks.append("") return JsonResponse( From 4a3247e1674617cfec1366da01dcba81eb0e5f67 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 16 Sep 2025 14:22:15 +0300 Subject: [PATCH 39/49] Small refactoring --- src/indicatorsets/utils.py | 591 +++++++++++++++++++++++++++++++++++++ src/indicatorsets/views.py | 573 ++++++----------------------------- 2 files changed, 681 insertions(+), 483 deletions(-) diff --git a/src/indicatorsets/utils.py b/src/indicatorsets/utils.py index 4d5eba2..b07b267 100644 --- a/src/indicatorsets/utils.py +++ b/src/indicatorsets/utils.py @@ -2,13 +2,18 @@ import random from collections import defaultdict from datetime import datetime as dtime +from textwrap import dedent import requests +from delphi_utils import get_structured_logger from django.conf import settings +from django.http import JsonResponse from epiweeks import Week from indicatorsets.models import IndicatorSet +FLUVIEW_INDICATORS_MAPPING = {"wili": "%wILI", "ili": "%ILI"} + def list_to_dict(lst): result = {} @@ -81,3 +86,589 @@ def group_by_property(list_of_dicts, property): for item in list_of_dicts: grouped_dict[item[property]].append(item) return dict(grouped_dict) + + +def generate_covidcast_dataset_epivis(indicator, covidcast_geos, api_key): + datasets = [] + for geo in covidcast_geos: + if geo["id"] not in indicator.get("notCoveredGeos", []): + geo_value = ( + geo["id"].split(":")[1].lower() + if geo["geoType"] in ["nation", "state"] + else geo["id"].split(":")[1] + ) + datasets.append( + { + "color": generate_random_color(), + "title": "value", + "params": { + "_endpoint": indicator["_endpoint"], + "data_source": indicator["data_source"], + "signal": indicator["indicator"], + "time_type": indicator["time_type"], + "geo_type": geo["geoType"], + "geo_value": geo_value, + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) + get_structured_logger("form_activity_logger").info( + mode="epivis", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_type=geo["geoType"], + geo_value=geo_value, + api_key=api_key, + ) # noqa: E501 + if datasets: + return datasets + + +def generate_fluview_dataset_epivis(indicator, fluview_geos, api_key): + datasets = [] + for geo in fluview_geos: + datasets.append( + { + "color": generate_random_color(), + "title": FLUVIEW_INDICATORS_MAPPING.get( + indicator["indicator"], indicator["indicator"] + ), + "params": { + "_endpoint": ( + indicator["_endpoint"] + if indicator["data_source"] == "fluview" + else "fluview_clinical" + ), + "regions": geo["id"], + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) + get_structured_logger("form_activity_logger").info( + mode="epivis", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_value=geo["id"], + api_key=api_key, + ) # noqa: E501 + if datasets: + return datasets + + +def generate_nidss_flu_dataset_epivis(indicator, nidss_flu_geos, api_key): + datasets = [] + for geo in nidss_flu_geos: + datasets.append( + { + "color": generate_random_color(), + "title": indicator["indicator"], + "params": { + "_endpoint": indicator["_endpoint"], + "regions": geo["id"], + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) + get_structured_logger("form_activity_logger").info( + mode="epivis", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_value=geo["id"], + api_key=api_key, + ) + + +def generate_nidss_dengue_dataset_epivis(indicator, nidss_dengue_geos, api_key): + datasets = [] + for geo in nidss_dengue_geos: + datasets.append( + { + "color": generate_random_color(), + "title": indicator["indicator"], + "params": { + "_endpoint": indicator["_endpoint"], + "locations": geo["id"], + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) + get_structured_logger.info( + mode="epivis", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_value=geo["id"], + api_key=api_key, + ) + + +def generate_covidcast_indicators_export_url( + indicators, start_date, end_date, covidcast_geos, api_key +): + data_export_commands = [] + for indicator in indicators: + if indicator["_endpoint"] == "covidcast": + dates = get_epiweek(start_date, end_date) if indicator["time_type"] == "week" else [start_date, end_date] # fmt: skip + for type, values in covidcast_geos.items(): + geo_values = ",".join( + [ + ( + value["id"].split(":")[1].lower() + if value["geoType"] in ["nation", "state"] + else value["id"].split(":")[1] + ) + for value in values + ] + ) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_type=type, + geo_value=geo_values, + api_key=api_key, + ) # noqa: E501 + data_export_url = f"{settings.EPIDATA_URL}covidcast/csv?signal={indicator['data_source']}:{indicator['indicator']}&start_day={dates[0]}&end_day={dates[1]}&geo_type={type}&geo_values={geo_values}" + if api_key: + data_export_url += f"&api_key={api_key}" + data_export_commands.append( + f'wget --content-disposition {data_export_url}' + ) + return data_export_commands + + +def generate_fluview_indicators_export_url(fluview_geos, start_date, end_date, api_key): + data_export_commands = [] + regions = ",".join([region["id"] for region in fluview_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="fluview", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + data_export_url = f"{settings.EPIDATA_URL}fluview/?regions={regions}&epiweeks={date_from}-{date_to}&format=csv" + if api_key: + data_export_url += f"&api_key={api_key}" + data_export_commands.append( + f'wget --content-disposition {data_export_url}' + ) + return data_export_commands + + +def generate_nidss_flu_export_url(nidss_flu_geos, start_date, end_date, api_key): + data_export_commands = [] + regions = ",".join([region["id"] for region in nidss_flu_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="nidss_flu", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + data_export_url = f"{settings.EPIDATA_URL}nidss_flu/?regions={regions}&epiweeks={date_from}-{date_to}&format=csv" + if api_key: + data_export_url += f"&api_key={api_key}" + data_export_commands.append( + f'wget --content-disposition {data_export_url}' + ) + return data_export_commands + + +def generate_nidss_dengue_export_url(nidss_dengue_geos, start_date, end_date, api_key): + data_export_commands = [] + regions = ",".join([region["id"] for region in nidss_dengue_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="nidss_dengue", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + data_export_url = f"{settings.EPIDATA_URL}nidss_dengue/?locations={regions}&epiweeks={date_from}-{date_to}&format=csv" # fmt: skip + if api_key: + data_export_url += f"&api_key={api_key}" + data_export_commands.append( + f'wget --content-disposition {data_export_url}' + ) + return data_export_commands + + +def preview_covidcast_data(indicators, start_date, end_date, covidcast_geos, api_key): + preview_data = [] + for indicator in indicators: + if indicator["_endpoint"] == "covidcast": + time_values = f"{start_date}--{end_date}" + if indicator["time_type"] == "week": + start_day, end_day = get_epiweek(start_date, end_date) + time_values = f"{start_day}-{end_day}" + for geo_type, values in covidcast_geos.items(): + geo_values = ",".join( + [ + ( + value["id"].split(":")[1].lower() + if value["geoType"] in ["nation", "state"] + else value["id"].split(":")[1] + ) + for value in values + ] + ) + params = { + "time_type": indicator["time_type"], + "time_values": time_values, + "data_source": indicator["data_source"], + "signal": indicator["indicator"], + "geo_type": geo_type, + "geo_values": geo_values, + "api_key": api_key if api_key else settings.EPIDATA_API_KEY, + } + get_structured_logger("form_activity_logger").info( + mode="preview_data", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_type=geo_type, + geo_value=geo_values, + api_key=api_key, + ) # noqa: E501 + response = requests.get( + f"{settings.EPIDATA_URL}covidcast", params=params + ) + if response.status_code == 200: + data = response.json() + if len(data["epidata"]): + preview_data.append( + { + "epidata": data["epidata"][0], + "result": data["result"], + "message": data["message"], + } + ) + elif response.status_code == 401: + preview_data = { + "epidata": [], + "result": -2, + "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", + } + return JsonResponse(preview_data, safe=False) + return preview_data + + +def preview_fluview_data(fluview_geos, start_date, end_date, api_key): + preview_data = [] + regions = ",".join([region["id"] for region in fluview_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + params = { + "regions": regions, + "epiweeks": f"{date_from}-{date_to}", + "api_key": api_key if api_key else settings.EPIDATA_API_KEY, + } + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="fluview", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + response = requests.get(f"{settings.EPIDATA_URL}fluview", params=params) + if response.status_code == 200: + data = response.json() + if len(data["epidata"]): + preview_data.append( + { + "epidata": data["epidata"][0], + "result": data["result"], + "message": data["message"], + } + ) + elif response.status_code == 401: + preview_data = { + "epidata": [], + "result": -2, + "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", + } + return JsonResponse(preview_data, safe=False) + return preview_data + + +def preview_nidss_flu_data(nidss_flu_geos, start_date, end_date, api_key): + preview_data = [] + regions = ",".join([region["id"] for region in nidss_flu_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + params = { + "regions": regions, + "epiweeks": f"{date_from}-{date_to}", + "api_key": api_key if api_key else settings.EPIDATA_API_KEY, + } + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="nidss_flu", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + response = requests.get(f"{settings.EPIDATA_URL}nidss_flu", params=params) + if response.status_code == 200: + data = response.json() + if len(data["epidata"]): + preview_data.append( + { + "epidata": data["epidata"][0], + "result": data["result"], + "message": data["message"], + } + ) + elif response.status_code == 401: + preview_data = { + "epidata": [], + "result": -2, + "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", + } + return JsonResponse(preview_data, safe=False) + + +def preview_nidss_dengue_data(nidss_dengue_geos, start_date, end_date, api_key): + preview_data = [] + regions = ",".join([region["id"] for region in nidss_dengue_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + params = { + "locations": regions, + "epiweeks": f"{date_from}-{date_to}", + "api_key": api_key if api_key else settings.EPIDATA_API_KEY, + } + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="nidss_dengue", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + response = requests.get(f"{settings.EPIDATA_URL}nidss_dengue", params=params) + if response.status_code == 200: + data = response.json() + if len(data["epidata"]): + preview_data.append( + { + "epidata": data["epidata"][0], + "result": data["result"], + "message": data["message"], + } + ) + elif response.status_code == 401: + preview_data = { + "epidata": [], + "result": -2, + "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", + } + return JsonResponse(preview_data, safe=False) + return preview_data + + +def generate_query_code_covidcast( + indicators, + covidcast_geos, + start_date, + end_date, + data_source, + indicators_str, + api_key, +): + python_code_blocks = [] + r_code_blocks = [] + time_type = indicators[0].get("time_type") + for geo_type, values in covidcast_geos.items(): + geo_values = [ + ( + value["id"].split(":")[1].lower() + if value["geoType"] in ["nation", "state"] + else value["id"].split(":")[1] + ) + for value in values + ] + for indicator in indicators: + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="covidcast", + data_source=data_source, + indicator=indicator["indicator"], + geo_type=geo_type, + geo_value=",".join(geo_values), + api_key=api_key, + ) # noqa: E501 + if time_type == "week": + start_week, end_week = get_epiweek(start_date, end_date) + python_code_block = dedent( + f"""\ + {data_source.replace('-', '_')}_{geo_type}_df = epidata.pub_covidcast( + data_source="{data_source}", + signals="{indicators_str}", + geo_type="{geo_type}", + time_type="{time_type}", + geo_values="{','.join(geo_values)}", + time_values=EpiRange({start_week}, {end_week}), + ).df() + """ + ) + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_{data_source.replace("-", "_")}_{geo_type} <- pub_covidcast( + source = "{data_source}", + signals = "{indicators_str}", + geo_type = "{geo_type}", + time_type = "{time_type}", + geo_values = "{','.join(geo_values)}", + time_values = epirange({start_week}, {end_week}) + ) + """ + ) + r_code_blocks.append(r_code_block) + else: + python_code_block = dedent( + f"""\ + {data_source.replace('-', '_')}_{geo_type}_df = epidata.pub_covidcast( + data_source="{data_source}", + signals="{indicators_str}", + geo_type="{geo_type}", + time_type="{time_type}", + geo_values="{','.join(geo_values)}", + time_values=EpiRange({start_date.replace("-", "")}, {end_date.replace("-", "")}), + ).df() + """ + ) + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_{data_source.replace("-", "_")}_{geo_type} <- pub_covidcast( + source = "{data_source}", + signals = "{indicators_str}", + geo_type = "{geo_type}", + time_type = "{time_type}", + geo_values = "{','.join(geo_values)}", + time_values = epirange({start_date.replace("-", "")}, {end_date.replace("-", "")}) + ) + """ + ) + r_code_blocks.append(r_code_block) + return python_code_blocks, r_code_blocks + + +def generate_query_code_fluview( + fluview_geos, start_date, end_date, data_source, api_key +): + python_code_blocks = [] + r_code_blocks = [] + regions = ",".join([region["id"] for region in fluview_geos]) + start_week, end_week = get_epiweek(start_date, end_date) + python_code_block = dedent( + f"""\ + fluview_df = epidata.pub_fluview( + regions="{regions}", + epiweeks="{start_week}-{end_week}", + ).df() + """ + ) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="fluview", + regions=regions, + epiweeks=f"{start_week}-{end_week}", + api_key=api_key, + ) # noqa: E501 + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_{data_source.replace("-", "_")} <- pub_fluview( + regions = "{regions}", + epiweeks = epirange({start_week}, {end_week}) + ) + """ + ) + r_code_blocks.append(r_code_block) + return python_code_blocks, r_code_blocks + + +def generate_query_code_nidss_flu( + nidss_flu_geos, start_date, end_date, data_source, api_key +): + python_code_blocks = [] + r_code_blocks = [] + regions = ",".join([region["id"] for region in nidss_flu_geos]) + start_week, end_week = get_epiweek(start_date, end_date) + python_code_block = dedent( + f"""\ + nidss_flu_df = epidata.pub_nidss_flu( + regions="{regions}", + epiweeks="{start_week}-{end_week}", + ).df() + """ + ) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="nidss_flu", + regions=regions, + epiweeks=f"{start_week}-{end_week}", + api_key=api_key, + ) # noqa: E501 + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_{data_source.replace("-", "_")} <- pub_nidss_flu( + regions = "{regions}", + epiweeks = epirange({start_week}, {end_week}) + ) + """ + ) + r_code_blocks.append(r_code_block) + return python_code_blocks, r_code_blocks + + +def generate_query_code_nidss_dengue( + nidss_dengue_geos, start_date, end_date, data_source, api_key +): + python_code_blocks = [] + r_code_blocks = [] + regions = ",".join([region["id"] for region in nidss_dengue_geos]) + start_week, end_week = get_epiweek(start_date, end_date) + python_code_block = dedent( + f"""\ + nidss_dengue_df = epidata.pub_nidss_dengue( + locations="{regions}", + epiweeks="{start_week}-{end_week}", + ).df() + """ + ) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="nidss_dengue", + regions=regions, + epiweeks=f"{start_week}-{end_week}", + api_key=api_key, + ) # noqa: E501 + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_{data_source.replace("-", "_")} <- pub_nidss_dengue( + locations = "{regions}", + epiweeks = epirange({start_week}, {end_week}) + ) + """ + ) + r_code_blocks.append(r_code_block) + + return python_code_blocks, r_code_blocks diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index b14da09..e2f6664 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -14,10 +14,23 @@ from indicatorsets.forms import IndicatorSetFilterForm from indicatorsets.models import IndicatorSet, FilterDescription, ColumnDescription from indicatorsets.utils import ( - generate_epivis_custom_title, - generate_random_color, - get_epiweek, group_by_property, + generate_covidcast_dataset_epivis, + generate_fluview_dataset_epivis, + generate_nidss_flu_dataset_epivis, + generate_nidss_dengue_dataset_epivis, + generate_covidcast_indicators_export_url, + generate_fluview_indicators_export_url, + generate_nidss_flu_export_url, + generate_nidss_dengue_export_url, + preview_covidcast_data, + preview_fluview_data, + preview_nidss_flu_data, + preview_nidss_dengue_data, + generate_query_code_covidcast, + generate_query_code_fluview, + generate_query_code_nidss_flu, + generate_query_code_nidss_dengue, ) from delphi_utils import get_structured_logger @@ -28,8 +41,6 @@ HEADER_DESCRIPTION = "Discover, display and download real-time infectious disease indicators (time series) that track a variety of pathogens, diseases and syndromes in a variety of locations (primarily within the USA). Browse the list, or filter it first by locations and pathogens of interest, by surveillance categories, and more. Expand any row to expose and select from a set of related indicators, then hit 'Show Selected Indicators' at bottom to plot or export your selected indicators, or to generate code snippets to retrieve them from the Delphi Epidata API. Most indicators are served from the Delphi Epidata real-time repository, but some may be available only from third parties or may require prior approval." -FLUVIEW_INDICATORS_MAPPING = {"wili": "%wILI", "ili": "%ILI"} - class IndicatorSetListView(ListView): model = IndicatorSet @@ -247,115 +258,27 @@ def epivis(request): ) # noqa: E501 for indicator in indicators: if indicator["_endpoint"] == "covidcast": - for geo in covidcast_geos: - if geo["id"] not in indicator.get("notCoveredGeos", []): - geo_value = ( - geo["id"].split(":")[1].lower() - if geo["geoType"] in ["nation", "state"] - else geo["id"].split(":")[1] - ) - datasets.append( - { - "color": generate_random_color(), - "title": "value", - "params": { - "_endpoint": indicator["_endpoint"], - "data_source": indicator["data_source"], - "signal": indicator["indicator"], - "time_type": indicator["time_type"], - "geo_type": geo["geoType"], - "geo_value": geo_value, - "custom_title": generate_epivis_custom_title( - indicator, geo["text"] - ), - }, - } - ) - form_activity_logger.info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_type=geo["geoType"], - geo_value=geo_value, - api_key=api_key, - ) # noqa: E501 - elif indicator["_endpoint"] == "fluview": - for geo in fluview_geos: - datasets.append( - { - "color": generate_random_color(), - "title": FLUVIEW_INDICATORS_MAPPING.get( - indicator["indicator"], indicator["indicator"] - ), - "params": { - "_endpoint": ( - indicator["_endpoint"] - if indicator["data_source"] == "fluview" - else "fluview_clinical" - ), - "regions": geo["id"], - "custom_title": generate_epivis_custom_title( - indicator, geo["text"] - ), - }, - } + datasets.extend( + generate_covidcast_dataset_epivis( + indicator, covidcast_geos, api_key ) - form_activity_logger.info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_value=geo["id"], - api_key=api_key, - ) # noqa: E501 - + ) + elif indicator["_endpoint"] == "fluview": + datasets.extend( + generate_fluview_dataset_epivis(indicator, fluview_geos, api_key) + ) elif indicator["_endpoint"] == "nidss_flu": - for geo in nidss_flu_locations: - datasets.append( - { - "color": generate_random_color(), - "title": indicator["indicator"], - "params": { - "_endpoint": indicator["_endpoint"], - "regions": geo["id"], - "custom_title": generate_epivis_custom_title( - indicator, geo["text"] - ), - }, - } - ) - form_activity_logger.info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_value=geo["id"], - api_key=api_key, + datasets.extend( + generate_nidss_flu_dataset_epivis( + indicator, nidss_flu_locations, api_key ) + ) elif indicator["_endpoint"] == "nidss_dengue": - for geo in nidss_dengue_locations: - datasets.append( - { - "color": generate_random_color(), - "title": indicator["indicator"], - "params": { - "_endpoint": indicator["_endpoint"], - "locations": geo["id"], - "custom_title": generate_epivis_custom_title( - indicator, geo["text"] - ), - }, - } - ) - form_activity_logger.info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_value=geo["id"], - api_key=api_key, + datasets.extend( + generate_nidss_dengue_dataset_epivis( + indicator, nidss_dengue_locations, api_key ) + ) if datasets: datasets_json = json.dumps({"datasets": datasets}) datasets_b64 = base64.b64encode(datasets_json.encode("ascii")).decode( @@ -386,82 +309,28 @@ def generate_export_data_url(request): fluview_geos_count=len(fluview_geos), api_key=api_key, ) # noqa: E501 - for indicator in indicators: - if indicator["_endpoint"] == "covidcast": - dates = get_epiweek(start_date, end_date) if indicator["time_type"] == "week" else [start_date, end_date] # fmt: skip - for type, values in covidcast_geos.items(): - geo_values = ",".join( - [ - ( - value["id"].split(":")[1].lower() - if value["geoType"] in ["nation", "state"] - else value["id"].split(":")[1] - ) - for value in values - ] - ) - form_activity_logger.info( - mode="data_export", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_type=type, - geo_value=geo_values, - api_key=api_key, - ) # noqa: E501 - data_export_url = f"{settings.EPIDATA_URL}covidcast/csv?signal={indicator['data_source']}:{indicator['indicator']}&start_day={dates[0]}&end_day={dates[1]}&geo_type={type}&geo_values={geo_values}" - if api_key: - data_export_url += f"&api_key={api_key}" - data_export_commands.append( - f'wget --content-disposition {data_export_url}' - ) + data_export_commands.extend( + generate_covidcast_indicators_export_url( + indicators, start_date, end_date, covidcast_geos, api_key + ) + ) if fluview_geos: - regions = ",".join([region["id"] for region in fluview_geos]) - date_from, date_to = get_epiweek(start_date, end_date) - form_activity_logger.info( - mode="data_export", - endpoint="fluview", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 - data_export_url = f"{settings.EPIDATA_URL}fluview/?regions={regions}&epiweeks={date_from}-{date_to}&format=csv" - if api_key: - data_export_url += f"&api_key={api_key}" - data_export_commands.append( - f'wget --content-disposition {data_export_url}' + data_export_commands.extend( + generate_fluview_indicators_export_url( + indicators, start_date, end_date, fluview_geos, api_key + ) ) if nidss_flu_locations: - regions = ",".join([region["id"] for region in nidss_flu_locations]) - date_from, date_to = get_epiweek(start_date, end_date) - form_activity_logger.info( - mode="data_export", - endpoint="nidss_flu", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 - data_export_url = f"{settings.EPIDATA_URL}nidss_flu/?regions={regions}&epiweeks={date_from}-{date_to}&format=csv" - if api_key: - data_export_url += f"&api_key={api_key}" - data_export_commands.append( - f'wget --content-disposition {data_export_url}' + data_export_commands.extend( + generate_nidss_flu_export_url( + indicators, start_date, end_date, nidss_flu_locations, api_key + ) ) if nidss_dengue_locations: - regions = ",".join([region["id"] for region in nidss_dengue_locations]) - date_from, date_to = get_epiweek(start_date, end_date) - form_activity_logger.info( - mode="data_export", - endpoint="nidss_dengue", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 - data_export_url = f"{settings.EPIDATA_URL}nidss_dengue/?locations={regions}&epiweeks={date_from}-{date_to}&format=csv" # fmt: skip - if api_key: - data_export_url += f"&api_key={api_key}" - data_export_commands.append( - f'wget --content-disposition {data_export_url}' + data_export_commands.extend( + generate_nidss_dengue_export_url( + indicators, start_date, end_date, nidss_dengue_locations, api_key + ) ) data_export_block = data_export_block.format("
".join(data_export_commands)) response = { @@ -491,159 +360,27 @@ def preview_data(request): fluview_geos_count=len(fluview_geos), api_key=api_key, ) # noqa: E501 - for indicator in indicators: - if indicator["_endpoint"] == "covidcast": - time_values = f"{start_date}--{end_date}" - if indicator["time_type"] == "week": - start_day, end_day = get_epiweek(start_date, end_date) - time_values = f"{start_day}-{end_day}" - for geo_type, values in covidcast_geos.items(): - geo_values = ",".join( - [ - ( - value["id"].split(":")[1].lower() - if value["geoType"] in ["nation", "state"] - else value["id"].split(":")[1] - ) - for value in values - ] - ) - params = { - "time_type": indicator["time_type"], - "time_values": time_values, - "data_source": indicator["data_source"], - "signal": indicator["indicator"], - "geo_type": geo_type, - "geo_values": geo_values, - "api_key": api_key if api_key else settings.EPIDATA_API_KEY, - } - form_activity_logger.info( - mode="preview_data", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_type=geo_type, - geo_value=geo_values, - api_key=api_key, - ) # noqa: E501 - response = requests.get( - f"{settings.EPIDATA_URL}covidcast", params=params - ) - if response.status_code == 200: - data = response.json() - if len(data["epidata"]): - preview_data.append( - { - "epidata": data["epidata"][0], - "result": data["result"], - "message": data["message"], - } - ) - elif response.status_code == 401: - preview_data = { - "epidata": [], - "result": -2, - "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", - } - return JsonResponse(preview_data, safe=False) + preview_data.extend( + preview_covidcast_data( + indicators, start_date, end_date, covidcast_geos, api_key + ) + ) if fluview_geos: - regions = ",".join([region["id"] for region in fluview_geos]) - date_from, date_to = get_epiweek(start_date, end_date) - params = { - "regions": regions, - "epiweeks": f"{date_from}-{date_to}", - "api_key": api_key if api_key else settings.EPIDATA_API_KEY, - } - form_activity_logger.info( - mode="data_export", - endpoint="fluview", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 - response = requests.get(f"{settings.EPIDATA_URL}fluview", params=params) - if response.status_code == 200: - data = response.json() - if len(data["epidata"]): - preview_data.append( - { - "epidata": data["epidata"][0], - "result": data["result"], - "message": data["message"], - } - ) - elif response.status_code == 401: - preview_data = { - "epidata": [], - "result": -2, - "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", - } - return JsonResponse(preview_data, safe=False) + preview_data.extend( + preview_fluview_data(fluview_geos, start_date, end_date, api_key) + ) if nidss_flu_locations: - regions = ",".join([region["id"] for region in nidss_flu_locations]) - date_from, date_to = get_epiweek(start_date, end_date) - params = { - "regions": regions, - "epiweeks": f"{date_from}-{date_to}", - "api_key": api_key if api_key else settings.EPIDATA_API_KEY, - } - form_activity_logger.info( - mode="data_export", - endpoint="nidss_flu", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 - response = requests.get(f"{settings.EPIDATA_URL}nidss_flu", params=params) - if response.status_code == 200: - data = response.json() - if len(data["epidata"]): - preview_data.append( - { - "epidata": data["epidata"][0], - "result": data["result"], - "message": data["message"], - } - ) - elif response.status_code == 401: - preview_data = { - "epidata": [], - "result": -2, - "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", - } - return JsonResponse(preview_data, safe=False) + preview_data.extend( + preview_nidss_flu_data( + nidss_flu_locations, start_date, end_date, api_key + ) + ) if nidss_dengue_locations: - regions = ",".join([region["id"] for region in nidss_dengue_locations]) - date_from, date_to = get_epiweek(start_date, end_date) - params = { - "locations": regions, - "epiweeks": f"{date_from}-{date_to}", - "api_key": api_key if api_key else settings.EPIDATA_API_KEY, - } - form_activity_logger.info( - mode="data_export", - endpoint="nidss_dengue", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 - response = requests.get(f"{settings.EPIDATA_URL}nidss_dengue", params=params) - if response.status_code == 200: - data = response.json() - if len(data["epidata"]): - preview_data.append( - { - "epidata": data["epidata"][0], - "result": data["result"], - "message": data["message"], - } - ) - elif response.status_code == 401: - preview_data = { - "epidata": [], - "result": -2, - "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", - } + preview_data.extend( + preview_nidss_dengue_data( + nidss_dengue_locations, start_date, end_date, api_key + ) + ) return JsonResponse(preview_data, safe=False) @@ -683,165 +420,35 @@ def create_query_code(request): [indicator["indicator"] for indicator in indicators] ) if indicators[0].get("_endpoint") == "covidcast": - time_type = indicators[0].get("time_type") - for geo_type, values in covidcast_geos.items(): - geo_values = [ - ( - value["id"].split(":")[1].lower() - if value["geoType"] in ["nation", "state"] - else value["id"].split(":")[1] - ) - for value in values - ] - for indicator in indicators: - form_activity_logger.info( - mode="data_export", - endpoint="covidcast", - data_source=data_source, - indicator=indicator["indicator"], - geo_type=geo_type, - geo_value=",".join(geo_values), - api_key=api_key, - ) # noqa: E501 - if time_type == "week": - start_week, end_week = get_epiweek(start_date, end_date) - python_code_block = dedent( - f"""\ - {data_source.replace('-', '_')}_{geo_type}_df = epidata.pub_covidcast( - data_source="{data_source}", - signals="{indicators_str}", - geo_type="{geo_type}", - time_type="{time_type}", - geo_values="{','.join(geo_values)}", - time_values=EpiRange({start_week}, {end_week}), - ).df() - """ - ) - python_code_blocks.append(python_code_block) - r_code_block = dedent( - f"""\ - epidata_{data_source.replace("-", "_")}_{geo_type} <- pub_covidcast( - source = "{data_source}", - signals = "{indicators_str}", - geo_type = "{geo_type}", - time_type = "{time_type}", - geo_values = "{','.join(geo_values)}", - time_values = epirange({start_week}, {end_week}) - ) - """ - ) - r_code_blocks.append(r_code_block) - else: - python_code_block = dedent( - f"""\ - {data_source.replace('-', '_')}_{geo_type}_df = epidata.pub_covidcast( - data_source="{data_source}", - signals="{indicators_str}", - geo_type="{geo_type}", - time_type="{time_type}", - geo_values="{','.join(geo_values)}", - time_values=EpiRange({start_date.replace("-", "")}, {end_date.replace("-", "")}), - ).df() - """ - ) - python_code_blocks.append(python_code_block) - r_code_block = dedent( - f"""\ - epidata_{data_source.replace("-", "_")}_{geo_type} <- pub_covidcast( - source = "{data_source}", - signals = "{indicators_str}", - geo_type = "{geo_type}", - time_type = "{time_type}", - geo_values = "{','.join(geo_values)}", - time_values = epirange({start_date.replace("-", "")}, {end_date.replace("-", "")}) - ) - """ - ) - r_code_blocks.append(r_code_block) - if fluview_geos: - regions = ",".join([region["id"] for region in fluview_geos]) - start_week, end_week = get_epiweek(start_date, end_date) - python_code_block = dedent( - f"""\ - fluview_df = epidata.pub_fluview( - regions="{regions}", - epiweeks="{start_week}-{end_week}", - ).df() - """ - ) - form_activity_logger.info( - mode="data_export", - endpoint="fluview", - regions=regions, - epiweeks=f"{start_week}-{end_week}", - api_key=api_key, - ) # noqa: E501 - python_code_blocks.append(python_code_block) - r_code_block = dedent( - f"""\ - epidata_{data_source.replace("-", "_")} <- pub_fluview( - regions = "{regions}", - epiweeks = epirange({start_week}, {end_week}) + python_code_block, r_code_block = generate_query_code_covidcast( + indicators, + covidcast_geos, + start_date, + end_date, + data_source, + indicators_str, + api_key, ) - """ + python_code_blocks.extend(python_code_block) + r_code_blocks.extend(r_code_block) + if fluview_geos: + python_code_block, r_code_block = generate_query_code_fluview( + fluview_geos, start_date, end_date, data_source, api_key ) - r_code_blocks.append(r_code_block) + python_code_blocks.extend(python_code_block) + r_code_blocks.extend(r_code_block) if nidss_flu_locations: - regions = ",".join([region["id"] for region in nidss_flu_locations]) - start_week, end_week = get_epiweek(start_date, end_date) - python_code_block = dedent( - f"""\ - nidss_flu_df = epidata.pub_nidss_flu( - regions="{regions}", - epiweeks="{start_week}-{end_week}", - ).df() - """ + python_code_block, r_code_block = generate_query_code_nidss_flu( + nidss_flu_locations, start_date, end_date, data_source, api_key ) - form_activity_logger.info( - mode="data_export", - endpoint="nidss_flu", - regions=regions, - epiweeks=f"{start_week}-{end_week}", - api_key=api_key, - ) # noqa: E501 - python_code_blocks.append(python_code_block) - r_code_block = dedent( - f"""\ - epidata_{data_source.replace("-", "_")} <- pub_nidss_flu( - regions = "{regions}", - epiweeks = epirange({start_week}, {end_week}) - ) - """ - ) - r_code_blocks.append(r_code_block) + python_code_blocks.extend(python_code_block) + r_code_blocks.extend(r_code_block) if nidss_dengue_locations: - regions = ",".join([region["id"] for region in nidss_dengue_locations]) - start_week, end_week = get_epiweek(start_date, end_date) - python_code_block = dedent( - f"""\ - nidss_dengue_df = epidata.pub_nidss_dengue( - locations="{regions}", - epiweeks="{start_week}-{end_week}", - ).df() - """ - ) - form_activity_logger.info( - mode="data_export", - endpoint="nidss_dengue", - regions=regions, - epiweeks=f"{start_week}-{end_week}", - api_key=api_key, - ) # noqa: E501 - python_code_blocks.append(python_code_block) - r_code_block = dedent( - f"""\ - epidata_{data_source.replace("-", "_")} <- pub_nidss_dengue( - locations = "{regions}", - epiweeks = epirange({start_week}, {end_week}) - ) - """ + python_code_block, r_code_block = generate_query_code_nidss_dengue( + nidss_dengue_locations, start_date, end_date, data_source, api_key ) - r_code_blocks.append(r_code_block) + python_code_blocks.extend(python_code_block) + r_code_blocks.extend(r_code_block) python_code_blocks.append("") r_code_blocks.append("") return JsonResponse( From 32cbc04cea3dd9f67f45bbd15f2b3bacf55eaf1b Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 16 Sep 2025 21:53:18 +0300 Subject: [PATCH 40/49] Added flusurv --- src/assets/js/indicatorHandler.js | 68 +++++++++--- src/assets/js/indicatorSetsTable.js | 3 +- src/assets/js/selectedIndicatorsModal.js | 16 +++ src/indicatorsets/utils.py | 132 +++++++++++++++++++++-- src/indicatorsets/views.py | 42 ++++++-- 5 files changed, 235 insertions(+), 26 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 80cb8a0..3b361a2 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -18,7 +18,7 @@ class IndicatorHandler { ili: "%ILI", }; - fluSurvRegions = [ + flusurvLocations = [ { id: "network_all", text: "Entire Network" }, { id: "network_eip", text: "EIP Netowrk" }, { id: "network_ihsp", text: "IHSP Network" }, @@ -214,6 +214,16 @@ class IndicatorHandler { return nidssDengueIndicators; } + getFlusurvIndicators() { + var flusurvIndicators = []; + this.indicators.forEach((indicator) => { + if (indicator["_endpoint"] === "flusurv") { + flusurvIndicators.push(indicator); + } + }); + return flusurvIndicators; + } + getFromToDate(startDate, endDate, timeType) { if (timeType === "week") { $.ajax({ @@ -306,18 +316,41 @@ class IndicatorHandler { } } + showFlusurvLocations() { + var flusurvLocationselect = ` +
+
+ +
+
+ +
+
`; + if ($("#otherEndpointLocations").length) { + $("#otherEndpointLocations").append(flusurvLocationselect); + $("#flusurvLocations").select2({ + placeholder: "Select FluSurv Location(s)", + data: this.flusurvLocations, + allowClear: true, + width: "100%", + }); + } + } + plotData() { const covidCastGeographicValues = $("#geographic_value").select2("data"); const fluviewLocations = $("#fluviewLocations").select2("data"); const nidssFluLocations = $("#nidssFluLocations").select2("data"); const nidssDengueLocations = $("#nidssDengueLocations").select2("data"); + const flusurvLocations = $("#flusurvLocations").select2("data"); const submitData = { indicators: this.indicators, covidCastGeographicValues: covidCastGeographicValues, fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, nidssDengueLocations: nidssDengueLocations, + flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, }; const csrftoken = Cookies.get("csrftoken"); @@ -338,6 +371,7 @@ class IndicatorHandler { fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), + flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), epivisUrl: data["epivis_url"], apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } @@ -347,9 +381,10 @@ class IndicatorHandler { } exportData() { - var fluviewLocations = $("#fluviewLocations").select2("data"); - var nidssFluLocations = $("#nidssFluLocations").select2("data"); - var nidssDengueLocations = $("#nidssDengueLocations").select2("data"); + const fluviewLocations = $("#fluviewLocations").select2("data"); + const nidssFluLocations = $("#nidssFluLocations").select2("data"); + const nidssDengueLocations = $("#nidssDengueLocations").select2("data"); + const flusurvLocations = $("#flusurvLocations").select2("data"); var covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), @@ -363,6 +398,7 @@ class IndicatorHandler { fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, nidssDengueLocations: nidssDengueLocations, + flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -385,6 +421,7 @@ class IndicatorHandler { fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), + flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -394,11 +431,12 @@ class IndicatorHandler { previewData() { $('#loader').show(); - var fluviewLocations = $("#fluviewLocations").select2("data"); - var nidssFluLocations = $("#nidssFluLocations").select2("data"); - var nidssDengueLocations = $("#nidssDengueLocations").select2("data"); + const fluviewLocations = $("#fluviewLocations").select2("data"); + const nidssFluLocations = $("#nidssFluLocations").select2("data"); + const nidssDengueLocations = $("#nidssDengueLocations").select2("data"); + const flusurvLocations = $("#flusurvLocations").select2("data"); - var covidCastGeographicValues = Object.groupBy( + const covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), ({ geoType }) => [geoType] ); @@ -410,6 +448,7 @@ class IndicatorHandler { fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, nidssDengueLocations: nidssDengueLocations, + flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -431,6 +470,7 @@ class IndicatorHandler { fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), + flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); @@ -440,12 +480,12 @@ class IndicatorHandler { } createQueryCode() { + const fluviewLocations = $("#fluviewLocations").select2("data"); + const nidssFluLocations = $("#nidssFluLocations").select2("data"); + const nidssDengueLocations = $("#nidssDengueLocations").select2("data"); + const flusurvLocations = $("#flusurvLocations").select2("data"); - var fluviewLocations = $("#fluviewLocations").select2("data"); - var nidssFluLocations = $("#nidssFluLocations").select2("data"); - var nidssDengueLocations = $("#nidssDengueLocations").select2("data"); - - var covidCastGeographicValues = Object.groupBy( + const covidCastGeographicValues = Object.groupBy( $("#geographic_value").select2("data"), ({ geoType }) => [geoType] ); @@ -458,6 +498,7 @@ class IndicatorHandler { fluviewLocations: fluviewLocations, nidssFluLocations: nidssFluLocations, nidssDengueLocations: nidssDengueLocations, + flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, } const csrftoken = Cookies.get("csrftoken"); @@ -487,6 +528,7 @@ class IndicatorHandler { fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), + flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", } dataLayerPush(payload); diff --git a/src/assets/js/indicatorSetsTable.js b/src/assets/js/indicatorSetsTable.js index 26ec369..9f0398b 100644 --- a/src/assets/js/indicatorSetsTable.js +++ b/src/assets/js/indicatorSetsTable.js @@ -83,7 +83,8 @@ function format(indicatorSetId, relatedIndicators, indicatorSetDescription) { ).length; var checkboxTitle = ""; checked = checked ? "checked" : ""; - disabled = indicator.endpoint !== "covidcast" && indicator.endpoint !== "fluview" && indicator.endpoint !== "nidss_flu" && indicator.endpoint !== "nidss_dengue" ? "disabled" : ""; + const enabledEndpoints = ["covidcast", "fluview", "nidss_flu", "nidss_dengue", "flusurv"]; + disabled = enabledEndpoints.includes(indicator.endpoint) ? "" : "disabled"; sourceType = indicator.source_type; var restricted = indicator.restricted != "No"; if (disabled === "disabled") { diff --git a/src/assets/js/selectedIndicatorsModal.js b/src/assets/js/selectedIndicatorsModal.js index d81c61e..8722246 100644 --- a/src/assets/js/selectedIndicatorsModal.js +++ b/src/assets/js/selectedIndicatorsModal.js @@ -269,6 +269,21 @@ function showNIDSSDengueLocationSelect() { } } +function showFlusurvLocationSelect() { + if (indicatorHandler.getFlusurvIndicators().length > 0) { + if (document.getElementsByName("flusurvLocations").length === 0) { + indicatorHandler.showFlusurvLocations(); + } else { + // IF code goes here, we assume that otherEndpointLocationWarning & flusurvRegion selector is already on the page, but is just hidden, so we should just show it. + $("#flusurvDiv").show(); + } + } else { + // If there are no non-covidcast indicators selected then hide otherEndpointLocationWarning & flusurvLocations selector. + $("#flusurvLocations").val(null).trigger("change"); + $("#flusurvDiv").hide(); + } +} + function showNonDelphiIndicatorSetsLocations() { if (indicatorHandler.nonCovidcastIndicatorSets.length > 0) { @@ -277,6 +292,7 @@ function showNonDelphiIndicatorSetsLocations() { showFluviewLocationSelect(); showNIDSSFluLocationSelect(); showNIDSSDengueLocationSelect(); + showFlusurvLocationSelect(); $("#otherEndpointLocationsWrapper").show(); } else { $("#differentLocationNote").html(""); diff --git a/src/indicatorsets/utils.py b/src/indicatorsets/utils.py index b07b267..8b82976 100644 --- a/src/indicatorsets/utils.py +++ b/src/indicatorsets/utils.py @@ -213,6 +213,33 @@ def generate_nidss_dengue_dataset_epivis(indicator, nidss_dengue_geos, api_key): ) +def generate_flusurv_dataset_epivis(indicator, flusurv_geos, api_key): + datasets = [] + for geo in flusurv_geos: + datasets.append( + { + "color": generate_random_color(), + "title": indicator["indicator"], + "params": { + "_endpoint": indicator["_endpoint"], + "locations": geo["id"], + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) + get_structured_logger("form_activity_logger").info( + mode="epivis", + endpoint=indicator["_endpoint"], + data_source=indicator["data_source"], + indicator=indicator["indicator"], + geo_value=geo["id"], + api_key=api_key, + ) + return datasets + + def generate_covidcast_indicators_export_url( indicators, start_date, end_date, covidcast_geos, api_key ): @@ -309,6 +336,26 @@ def generate_nidss_dengue_export_url(nidss_dengue_geos, start_date, end_date, ap return data_export_commands +def generate_flusurv_export_url(flusurv_geos, start_date, end_date, api_key): + data_export_commands = [] + regions = ",".join([region["id"] for region in flusurv_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="flusurv", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + data_export_url = f"{settings.EPIDATA_URL}flusurv/?locations={regions}&epiweeks={date_from}-{date_to}&format=csv" # fmt: skip + if api_key: + data_export_url += f"&api_key={api_key}" + data_export_commands.append( + f'wget --content-disposition {data_export_url}' + ) + return data_export_commands + + def preview_covidcast_data(indicators, start_date, end_date, covidcast_geos, api_key): preview_data = [] for indicator in indicators: @@ -479,6 +526,43 @@ def preview_nidss_dengue_data(nidss_dengue_geos, start_date, end_date, api_key): return preview_data +def preview_flusurv_data(flusurv_geos, start_date, end_date, api_key): + preview_data = [] + regions = ",".join([region["id"] for region in flusurv_geos]) + date_from, date_to = get_epiweek(start_date, end_date) + params = { + "locations": regions, + "epiweeks": f"{date_from}-{date_to}", + "api_key": api_key if api_key else settings.EPIDATA_API_KEY, + } + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="flusurv", + regions=regions, + epiweeks=f"{date_from}-{date_to}", + api_key=api_key, + ) # noqa: E501 + response = requests.get(f"{settings.EPIDATA_URL}flusurv", params=params) + if response.status_code == 200: + data = response.json() + if len(data["epidata"]): + preview_data.append( + { + "epidata": data["epidata"][0], + "result": data["result"], + "message": data["message"], + } + ) + elif response.status_code == 401: + preview_data = { + "epidata": [], + "result": -2, + "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", + } + return JsonResponse(preview_data, safe=False) + return preview_data + + def generate_query_code_covidcast( indicators, covidcast_geos, @@ -569,7 +653,7 @@ def generate_query_code_covidcast( def generate_query_code_fluview( - fluview_geos, start_date, end_date, data_source, api_key + fluview_geos, start_date, end_date, api_key ): python_code_blocks = [] r_code_blocks = [] @@ -593,7 +677,7 @@ def generate_query_code_fluview( python_code_blocks.append(python_code_block) r_code_block = dedent( f"""\ - epidata_{data_source.replace("-", "_")} <- pub_fluview( + epidata_fluview <- pub_fluview( regions = "{regions}", epiweeks = epirange({start_week}, {end_week}) ) @@ -604,7 +688,7 @@ def generate_query_code_fluview( def generate_query_code_nidss_flu( - nidss_flu_geos, start_date, end_date, data_source, api_key + nidss_flu_geos, start_date, end_date, api_key ): python_code_blocks = [] r_code_blocks = [] @@ -628,7 +712,7 @@ def generate_query_code_nidss_flu( python_code_blocks.append(python_code_block) r_code_block = dedent( f"""\ - epidata_{data_source.replace("-", "_")} <- pub_nidss_flu( + epidata_nidss_flu <- pub_nidss_flu( regions = "{regions}", epiweeks = epirange({start_week}, {end_week}) ) @@ -639,7 +723,7 @@ def generate_query_code_nidss_flu( def generate_query_code_nidss_dengue( - nidss_dengue_geos, start_date, end_date, data_source, api_key + nidss_dengue_geos, start_date, end_date, api_key ): python_code_blocks = [] r_code_blocks = [] @@ -663,7 +747,43 @@ def generate_query_code_nidss_dengue( python_code_blocks.append(python_code_block) r_code_block = dedent( f"""\ - epidata_{data_source.replace("-", "_")} <- pub_nidss_dengue( + epidata_nidss_dengue <- pub_nidss_dengue( + locations = "{regions}", + epiweeks = epirange({start_week}, {end_week}) + ) + """ + ) + r_code_blocks.append(r_code_block) + + return python_code_blocks, r_code_blocks + + +def generate_query_code_flusurv( + flusurv_geos, start_date, end_date, api_key +): + python_code_blocks = [] + r_code_blocks = [] + regions = ",".join([region["id"] for region in flusurv_geos]) + start_week, end_week = get_epiweek(start_date, end_date) + python_code_block = dedent( + f"""\ + flusurv_df = epidata.pub_flusurv( + locations="{regions}", + epiweeks="{start_week}-{end_week}", + ).df() + """ + ) + get_structured_logger("form_activity_logger").info( + mode="data_export", + endpoint="flusurv", + regions=regions, + epiweeks=f"{start_week}-{end_week}", + api_key=api_key, + ) # noqa: E501 + python_code_blocks.append(python_code_block) + r_code_block = dedent( + f"""\ + epidata_flusurv <- pub_flusurv( locations = "{regions}", epiweeks = epirange({start_week}, {end_week}) ) diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index e2f6664..d9ee9e0 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -19,18 +19,22 @@ generate_fluview_dataset_epivis, generate_nidss_flu_dataset_epivis, generate_nidss_dengue_dataset_epivis, + generate_flusurv_dataset_epivis, generate_covidcast_indicators_export_url, generate_fluview_indicators_export_url, generate_nidss_flu_export_url, generate_nidss_dengue_export_url, + generate_flusurv_export_url, preview_covidcast_data, preview_fluview_data, preview_nidss_flu_data, preview_nidss_dengue_data, + preview_flusurv_data, generate_query_code_covidcast, generate_query_code_fluview, generate_query_code_nidss_flu, generate_query_code_nidss_dengue, + generate_query_code_flusurv, ) from delphi_utils import get_structured_logger @@ -248,6 +252,7 @@ def epivis(request): fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) + flusurv_locations = data.get("flusurvLocations", []) api_key = data.get("apiKey", "") form_activity_logger.info( mode="epivis", @@ -279,6 +284,12 @@ def epivis(request): indicator, nidss_dengue_locations, api_key ) ) + elif indicator["_endpoint"] == "flusurv": + datasets.extend( + generate_flusurv_dataset_epivis( + indicator, flusurv_locations, api_key + ) + ) if datasets: datasets_json = json.dumps({"datasets": datasets}) datasets_b64 = base64.b64encode(datasets_json.encode("ascii")).decode( @@ -301,6 +312,7 @@ def generate_export_data_url(request): fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) + flusurv_locations = data.get("flusurvLocations", []) api_key = data.get("apiKey", None) form_activity_logger.info( mode="data_export", @@ -317,19 +329,25 @@ def generate_export_data_url(request): if fluview_geos: data_export_commands.extend( generate_fluview_indicators_export_url( - indicators, start_date, end_date, fluview_geos, api_key + fluview_geos, start_date, end_date, api_key ) ) if nidss_flu_locations: data_export_commands.extend( generate_nidss_flu_export_url( - indicators, start_date, end_date, nidss_flu_locations, api_key + nidss_flu_locations, start_date, end_date, api_key ) ) if nidss_dengue_locations: data_export_commands.extend( generate_nidss_dengue_export_url( - indicators, start_date, end_date, nidss_dengue_locations, api_key + nidss_dengue_locations, start_date, end_date, api_key + ) + ) + if flusurv_locations: + data_export_commands.extend( + generate_flusurv_export_url( + flusurv_locations, start_date, end_date, api_key ) ) data_export_block = data_export_block.format("
".join(data_export_commands)) @@ -350,6 +368,7 @@ def preview_data(request): fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) + flusurv_locations = data.get("flusurvLocations", []) api_key = data.get("apiKey", None) preview_data = [] @@ -381,6 +400,10 @@ def preview_data(request): nidss_dengue_locations, start_date, end_date, api_key ) ) + if flusurv_locations: + preview_data.extend( + preview_flusurv_data(flusurv_locations, start_date, end_date, api_key) + ) return JsonResponse(preview_data, safe=False) @@ -394,6 +417,7 @@ def create_query_code(request): fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) + flusurv_locations = data.get("flusurvLocations", []) api_key = data.get("apiKey", None) python_code_blocks = [ dedent( @@ -433,19 +457,25 @@ def create_query_code(request): r_code_blocks.extend(r_code_block) if fluview_geos: python_code_block, r_code_block = generate_query_code_fluview( - fluview_geos, start_date, end_date, data_source, api_key + fluview_geos, start_date, end_date, api_key ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) if nidss_flu_locations: python_code_block, r_code_block = generate_query_code_nidss_flu( - nidss_flu_locations, start_date, end_date, data_source, api_key + nidss_flu_locations, start_date, end_date, api_key ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) if nidss_dengue_locations: python_code_block, r_code_block = generate_query_code_nidss_dengue( - nidss_dengue_locations, start_date, end_date, data_source, api_key + nidss_dengue_locations, start_date, end_date, api_key + ) + python_code_blocks.extend(python_code_block) + r_code_blocks.extend(r_code_block) + if flusurv_locations: + python_code_block, r_code_block = generate_query_code_flusurv( + flusurv_locations, start_date, end_date, api_key ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) From 49c1c13c6478ac814623839e7ddc482badb587f4 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 16 Sep 2025 22:04:07 +0300 Subject: [PATCH 41/49] Version bump --- src/epiportal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index c7c0359..5b3e2e4 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.7" +APP_VERSION = "1.0.8" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From 1fb4a70b16780fedd3a9532620614dc9b741b788 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 22 Sep 2025 23:25:54 +0300 Subject: [PATCH 42/49] Improved logging --- src/indicatorsets/utils.py | 306 +++++++++++++++---------------------- src/indicatorsets/views.py | 52 ++----- 2 files changed, 137 insertions(+), 221 deletions(-) diff --git a/src/indicatorsets/utils.py b/src/indicatorsets/utils.py index 8b82976..921d4cb 100644 --- a/src/indicatorsets/utils.py +++ b/src/indicatorsets/utils.py @@ -3,15 +3,17 @@ from collections import defaultdict from datetime import datetime as dtime from textwrap import dedent +import json import requests -from delphi_utils import get_structured_logger from django.conf import settings from django.http import JsonResponse from epiweeks import Week +from delphi_utils import get_structured_logger from indicatorsets.models import IndicatorSet + FLUVIEW_INDICATORS_MAPPING = {"wili": "%wILI", "ili": "%ILI"} @@ -88,7 +90,7 @@ def group_by_property(list_of_dicts, property): return dict(grouped_dict) -def generate_covidcast_dataset_epivis(indicator, covidcast_geos, api_key): +def generate_covidcast_dataset_epivis(indicator, covidcast_geos): datasets = [] for geo in covidcast_geos: if geo["id"] not in indicator.get("notCoveredGeos", []): @@ -114,20 +116,10 @@ def generate_covidcast_dataset_epivis(indicator, covidcast_geos, api_key): }, } ) - get_structured_logger("form_activity_logger").info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_type=geo["geoType"], - geo_value=geo_value, - api_key=api_key, - ) # noqa: E501 - if datasets: - return datasets - - -def generate_fluview_dataset_epivis(indicator, fluview_geos, api_key): + return datasets + + +def generate_fluview_dataset_epivis(indicator, fluview_geos): datasets = [] for geo in fluview_geos: datasets.append( @@ -149,19 +141,10 @@ def generate_fluview_dataset_epivis(indicator, fluview_geos, api_key): }, } ) - get_structured_logger("form_activity_logger").info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_value=geo["id"], - api_key=api_key, - ) # noqa: E501 - if datasets: - return datasets - - -def generate_nidss_flu_dataset_epivis(indicator, nidss_flu_geos, api_key): + return datasets + + +def generate_nidss_flu_dataset_epivis(indicator, nidss_flu_geos): datasets = [] for geo in nidss_flu_geos: datasets.append( @@ -177,17 +160,10 @@ def generate_nidss_flu_dataset_epivis(indicator, nidss_flu_geos, api_key): }, } ) - get_structured_logger("form_activity_logger").info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_value=geo["id"], - api_key=api_key, - ) + return datasets -def generate_nidss_dengue_dataset_epivis(indicator, nidss_dengue_geos, api_key): +def generate_nidss_dengue_dataset_epivis(indicator, nidss_dengue_geos): datasets = [] for geo in nidss_dengue_geos: datasets.append( @@ -203,17 +179,10 @@ def generate_nidss_dengue_dataset_epivis(indicator, nidss_dengue_geos, api_key): }, } ) - get_structured_logger.info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_value=geo["id"], - api_key=api_key, - ) + return datasets -def generate_flusurv_dataset_epivis(indicator, flusurv_geos, api_key): +def generate_flusurv_dataset_epivis(indicator, flusurv_geos): datasets = [] for geo in flusurv_geos: datasets.append( @@ -229,14 +198,6 @@ def generate_flusurv_dataset_epivis(indicator, flusurv_geos, api_key): }, } ) - get_structured_logger("form_activity_logger").info( - mode="epivis", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_value=geo["id"], - api_key=api_key, - ) return datasets @@ -258,15 +219,6 @@ def generate_covidcast_indicators_export_url( for value in values ] ) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_type=type, - geo_value=geo_values, - api_key=api_key, - ) # noqa: E501 data_export_url = f"{settings.EPIDATA_URL}covidcast/csv?signal={indicator['data_source']}:{indicator['indicator']}&start_day={dates[0]}&end_day={dates[1]}&geo_type={type}&geo_values={geo_values}" if api_key: data_export_url += f"&api_key={api_key}" @@ -280,13 +232,6 @@ def generate_fluview_indicators_export_url(fluview_geos, start_date, end_date, a data_export_commands = [] regions = ",".join([region["id"] for region in fluview_geos]) date_from, date_to = get_epiweek(start_date, end_date) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="fluview", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 data_export_url = f"{settings.EPIDATA_URL}fluview/?regions={regions}&epiweeks={date_from}-{date_to}&format=csv" if api_key: data_export_url += f"&api_key={api_key}" @@ -300,13 +245,6 @@ def generate_nidss_flu_export_url(nidss_flu_geos, start_date, end_date, api_key) data_export_commands = [] regions = ",".join([region["id"] for region in nidss_flu_geos]) date_from, date_to = get_epiweek(start_date, end_date) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="nidss_flu", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 data_export_url = f"{settings.EPIDATA_URL}nidss_flu/?regions={regions}&epiweeks={date_from}-{date_to}&format=csv" if api_key: data_export_url += f"&api_key={api_key}" @@ -320,13 +258,6 @@ def generate_nidss_dengue_export_url(nidss_dengue_geos, start_date, end_date, ap data_export_commands = [] regions = ",".join([region["id"] for region in nidss_dengue_geos]) date_from, date_to = get_epiweek(start_date, end_date) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="nidss_dengue", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 data_export_url = f"{settings.EPIDATA_URL}nidss_dengue/?locations={regions}&epiweeks={date_from}-{date_to}&format=csv" # fmt: skip if api_key: data_export_url += f"&api_key={api_key}" @@ -340,13 +271,6 @@ def generate_flusurv_export_url(flusurv_geos, start_date, end_date, api_key): data_export_commands = [] regions = ",".join([region["id"] for region in flusurv_geos]) date_from, date_to = get_epiweek(start_date, end_date) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="flusurv", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 data_export_url = f"{settings.EPIDATA_URL}flusurv/?locations={regions}&epiweeks={date_from}-{date_to}&format=csv" # fmt: skip if api_key: data_export_url += f"&api_key={api_key}" @@ -384,15 +308,6 @@ def preview_covidcast_data(indicators, start_date, end_date, covidcast_geos, api "geo_values": geo_values, "api_key": api_key if api_key else settings.EPIDATA_API_KEY, } - get_structured_logger("form_activity_logger").info( - mode="preview_data", - endpoint=indicator["_endpoint"], - data_source=indicator["data_source"], - indicator=indicator["indicator"], - geo_type=geo_type, - geo_value=geo_values, - api_key=api_key, - ) # noqa: E501 response = requests.get( f"{settings.EPIDATA_URL}covidcast", params=params ) @@ -425,13 +340,6 @@ def preview_fluview_data(fluview_geos, start_date, end_date, api_key): "epiweeks": f"{date_from}-{date_to}", "api_key": api_key if api_key else settings.EPIDATA_API_KEY, } - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="fluview", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 response = requests.get(f"{settings.EPIDATA_URL}fluview", params=params) if response.status_code == 200: data = response.json() @@ -462,13 +370,6 @@ def preview_nidss_flu_data(nidss_flu_geos, start_date, end_date, api_key): "epiweeks": f"{date_from}-{date_to}", "api_key": api_key if api_key else settings.EPIDATA_API_KEY, } - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="nidss_flu", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 response = requests.get(f"{settings.EPIDATA_URL}nidss_flu", params=params) if response.status_code == 200: data = response.json() @@ -498,13 +399,6 @@ def preview_nidss_dengue_data(nidss_dengue_geos, start_date, end_date, api_key): "epiweeks": f"{date_from}-{date_to}", "api_key": api_key if api_key else settings.EPIDATA_API_KEY, } - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="nidss_dengue", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 response = requests.get(f"{settings.EPIDATA_URL}nidss_dengue", params=params) if response.status_code == 200: data = response.json() @@ -535,13 +429,6 @@ def preview_flusurv_data(flusurv_geos, start_date, end_date, api_key): "epiweeks": f"{date_from}-{date_to}", "api_key": api_key if api_key else settings.EPIDATA_API_KEY, } - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="flusurv", - regions=regions, - epiweeks=f"{date_from}-{date_to}", - api_key=api_key, - ) # noqa: E501 response = requests.get(f"{settings.EPIDATA_URL}flusurv", params=params) if response.status_code == 200: data = response.json() @@ -570,7 +457,6 @@ def generate_query_code_covidcast( end_date, data_source, indicators_str, - api_key, ): python_code_blocks = [] r_code_blocks = [] @@ -584,16 +470,6 @@ def generate_query_code_covidcast( ) for value in values ] - for indicator in indicators: - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="covidcast", - data_source=data_source, - indicator=indicator["indicator"], - geo_type=geo_type, - geo_value=",".join(geo_values), - api_key=api_key, - ) # noqa: E501 if time_type == "week": start_week, end_week = get_epiweek(start_date, end_date) python_code_block = dedent( @@ -652,9 +528,7 @@ def generate_query_code_covidcast( return python_code_blocks, r_code_blocks -def generate_query_code_fluview( - fluview_geos, start_date, end_date, api_key -): +def generate_query_code_fluview(fluview_geos, start_date, end_date): python_code_blocks = [] r_code_blocks = [] regions = ",".join([region["id"] for region in fluview_geos]) @@ -667,13 +541,6 @@ def generate_query_code_fluview( ).df() """ ) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="fluview", - regions=regions, - epiweeks=f"{start_week}-{end_week}", - api_key=api_key, - ) # noqa: E501 python_code_blocks.append(python_code_block) r_code_block = dedent( f"""\ @@ -687,9 +554,7 @@ def generate_query_code_fluview( return python_code_blocks, r_code_blocks -def generate_query_code_nidss_flu( - nidss_flu_geos, start_date, end_date, api_key -): +def generate_query_code_nidss_flu(nidss_flu_geos, start_date, end_date): python_code_blocks = [] r_code_blocks = [] regions = ",".join([region["id"] for region in nidss_flu_geos]) @@ -702,13 +567,6 @@ def generate_query_code_nidss_flu( ).df() """ ) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="nidss_flu", - regions=regions, - epiweeks=f"{start_week}-{end_week}", - api_key=api_key, - ) # noqa: E501 python_code_blocks.append(python_code_block) r_code_block = dedent( f"""\ @@ -722,9 +580,7 @@ def generate_query_code_nidss_flu( return python_code_blocks, r_code_blocks -def generate_query_code_nidss_dengue( - nidss_dengue_geos, start_date, end_date, api_key -): +def generate_query_code_nidss_dengue(nidss_dengue_geos, start_date, end_date): python_code_blocks = [] r_code_blocks = [] regions = ",".join([region["id"] for region in nidss_dengue_geos]) @@ -737,13 +593,6 @@ def generate_query_code_nidss_dengue( ).df() """ ) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="nidss_dengue", - regions=regions, - epiweeks=f"{start_week}-{end_week}", - api_key=api_key, - ) # noqa: E501 python_code_blocks.append(python_code_block) r_code_block = dedent( f"""\ @@ -758,9 +607,7 @@ def generate_query_code_nidss_dengue( return python_code_blocks, r_code_blocks -def generate_query_code_flusurv( - flusurv_geos, start_date, end_date, api_key -): +def generate_query_code_flusurv(flusurv_geos, start_date, end_date): python_code_blocks = [] r_code_blocks = [] regions = ",".join([region["id"] for region in flusurv_geos]) @@ -773,13 +620,6 @@ def generate_query_code_flusurv( ).df() """ ) - get_structured_logger("form_activity_logger").info( - mode="data_export", - endpoint="flusurv", - regions=regions, - epiweeks=f"{start_week}-{end_week}", - api_key=api_key, - ) # noqa: E501 python_code_blocks.append(python_code_block) r_code_block = dedent( f"""\ @@ -792,3 +632,109 @@ def generate_query_code_flusurv( r_code_blocks.append(r_code_block) return python_code_blocks, r_code_blocks + + +def get_client_ip(request): + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") + return ( + x_forwarded_for.split(",")[0] + if x_forwarded_for + else request.META.get("REMOTE_ADDR") + ) + + +def log_form_stats(request, form_mode): + data = json.loads(request.body) + log_data = { + "form_mode": form_mode, + "num_of_indicators": len(data.get("indicators", [])), + "num_of_covidcast_geos": len(data.get("covidCastGeographicValues", [])), + "num_of_fluview_geos": len(data.get("fluviewLocations", [])), + "num_of_nidss_flu_geos": len(data.get("nidssFluLocations", [])), + "num_of_nidss_dengue_geos": len(data.get("nidssDengueLocations", [])), + "num_of_flusurv_geos": len(data.get("flusurvLocations", [])), + "start_date": data.get("start_date"), + "end_date": data.get("end_date"), + "epiweeks": ( + get_epiweek(data.get("start_date"), data.get("end_date")) + if data.get("start_date") and data.get("end_date") + else [] + ), + "api_key_used": bool(data.get("api_key")), + "api_key": data.get("api_key", "")[:4] + "..." if data.get("api_key") else "", + "user_ip": get_client_ip(request), + } + + get_structured_logger("form_stats").info(log_data) + + +def log_form_data(request, form_mode): + + log_form_stats(request, form_mode) + + data = json.loads(request.body) + indicators = data.get("indicators", []) + indicators = [ + { + "endpoint": ind.get("_endpoint"), + "indicator": ind.get("indicator"), + "data_source": ind.get("data_source"), + "time_type": ind.get("time_type"), + + } for ind in indicators + ] # fmt: skip + indicators = group_by_property(indicators, "endpoint") + covidcast_geos = [ + { + "geo_type": geo.get("geoType"), + "geo_value": geo.get("id").split(":")[1], + "geo_text": geo.get("text"), + } for geo in data.get("covidCastGeographicValues", []) + ] # fmt: skip + fluview_geos = [ + { + "geo_value": geo.get("id"), + "geo_text": geo.get("text"), + } + for geo in data.get("fluviewLocations", []) + ] + nidss_flu_geos = [ + { + "geo_value": geo.get("id"), + "geo_text": geo.get("text"), + } + for geo in data.get("nidssFluLocations", []) + ] + nidss_dengue_geos = [ + { + "geo_value": geo.get("id"), + "geo_text": geo.get("text"), + } + for geo in data.get("nidssDengueLocations", []) + ] + flusurv_geos = [ + { + "geo_value": geo.get("id"), + "geo_text": geo.get("text"), + } + for geo in data.get("flusurvLocations", []) + ] + log_data = { + "mode": form_mode, + "indicators": [ + {"endpoint": endpoint, "indicators": group} + for endpoint, group in indicators.items() + ], + "covidcast_geos": covidcast_geos, + "fluview_geos": fluview_geos, + "nidss_flu_geos": nidss_flu_geos, + "nidss_dengue_geos": nidss_dengue_geos, + "flusurv_geos": flusurv_geos, + "start_date": data.get("start_date", ""), + "end_date": data.get("end_date", ""), + "epiweeks": get_epiweek(data.get("start_date", ""), data.get("end_date", "")) if data.get("start_date") and data.get("end_date") else [], # fmt: skip + "api_key_used": bool(data.get("api_key")), + "api_key": data.get("api_key", "")[:4] + "..." if data.get("api_key") else "", + "user_ip": get_client_ip(request), + } + get_structured_logger("form_activity_logger").info(log_data) diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index d9ee9e0..c1e94ff 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -35,14 +35,13 @@ generate_query_code_nidss_flu, generate_query_code_nidss_dengue, generate_query_code_flusurv, + log_form_data, ) from delphi_utils import get_structured_logger indicatorsets_logger = get_structured_logger("indicatorsets_logger") -form_activity_logger = get_structured_logger("form_activity_logger") - HEADER_DESCRIPTION = "Discover, display and download real-time infectious disease indicators (time series) that track a variety of pathogens, diseases and syndromes in a variety of locations (primarily within the USA). Browse the list, or filter it first by locations and pathogens of interest, by surveillance categories, and more. Expand any row to expose and select from a set of related indicators, then hit 'Show Selected Indicators' at bottom to plot or export your selected indicators, or to generate code snippets to retrieve them from the Delphi Epidata API. Most indicators are served from the Delphi Epidata real-time repository, but some may be available only from third parties or may require prior approval." @@ -253,42 +252,29 @@ def epivis(request): nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) flusurv_locations = data.get("flusurvLocations", []) - api_key = data.get("apiKey", "") - form_activity_logger.info( - mode="epivis", - indicator_count=len(indicators), - covidcast_geos_count=len(covidcast_geos), - fluview_geos_count=len(fluview_geos), - api_key=api_key, - ) # noqa: E501 + log_form_data(request, "epivis") for indicator in indicators: if indicator["_endpoint"] == "covidcast": datasets.extend( - generate_covidcast_dataset_epivis( - indicator, covidcast_geos, api_key - ) + generate_covidcast_dataset_epivis(indicator, covidcast_geos) ) elif indicator["_endpoint"] == "fluview": datasets.extend( - generate_fluview_dataset_epivis(indicator, fluview_geos, api_key) + generate_fluview_dataset_epivis(indicator, fluview_geos) ) elif indicator["_endpoint"] == "nidss_flu": datasets.extend( - generate_nidss_flu_dataset_epivis( - indicator, nidss_flu_locations, api_key - ) + generate_nidss_flu_dataset_epivis(indicator, nidss_flu_locations) ) elif indicator["_endpoint"] == "nidss_dengue": datasets.extend( generate_nidss_dengue_dataset_epivis( - indicator, nidss_dengue_locations, api_key + indicator, nidss_dengue_locations ) ) elif indicator["_endpoint"] == "flusurv": datasets.extend( - generate_flusurv_dataset_epivis( - indicator, flusurv_locations, api_key - ) + generate_flusurv_dataset_epivis(indicator, flusurv_locations) ) if datasets: datasets_json = json.dumps({"datasets": datasets}) @@ -314,13 +300,6 @@ def generate_export_data_url(request): nidss_dengue_locations = data.get("nidssDengueLocations", []) flusurv_locations = data.get("flusurvLocations", []) api_key = data.get("apiKey", None) - form_activity_logger.info( - mode="data_export", - indicator_count=len(indicators), - covidcast_geos_count=len(covidcast_geos), - fluview_geos_count=len(fluview_geos), - api_key=api_key, - ) # noqa: E501 data_export_commands.extend( generate_covidcast_indicators_export_url( indicators, start_date, end_date, covidcast_geos, api_key @@ -372,13 +351,6 @@ def preview_data(request): api_key = data.get("apiKey", None) preview_data = [] - form_activity_logger.info( - mode="preview_data", - indicator_count=len(indicators), - covidcast_geos_count=len(covidcast_geos), - fluview_geos_count=len(fluview_geos), - api_key=api_key, - ) # noqa: E501 preview_data.extend( preview_covidcast_data( indicators, start_date, end_date, covidcast_geos, api_key @@ -418,7 +390,6 @@ def create_query_code(request): nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) flusurv_locations = data.get("flusurvLocations", []) - api_key = data.get("apiKey", None) python_code_blocks = [ dedent( """\ @@ -451,31 +422,30 @@ def create_query_code(request): end_date, data_source, indicators_str, - api_key, ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) if fluview_geos: python_code_block, r_code_block = generate_query_code_fluview( - fluview_geos, start_date, end_date, api_key + fluview_geos, start_date, end_date ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) if nidss_flu_locations: python_code_block, r_code_block = generate_query_code_nidss_flu( - nidss_flu_locations, start_date, end_date, api_key + nidss_flu_locations, start_date, end_date ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) if nidss_dengue_locations: python_code_block, r_code_block = generate_query_code_nidss_dengue( - nidss_dengue_locations, start_date, end_date, api_key + nidss_dengue_locations, start_date, end_date ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) if flusurv_locations: python_code_block, r_code_block = generate_query_code_flusurv( - flusurv_locations, start_date, end_date, api_key + flusurv_locations, start_date, end_date ) python_code_blocks.extend(python_code_block) r_code_blocks.extend(r_code_block) From fdd35c9f657dfc293cd80bcd565e0bd8d9ab5a5d Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 22 Sep 2025 23:27:31 +0300 Subject: [PATCH 43/49] Changed function param --- src/indicatorsets/utils.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/indicatorsets/utils.py b/src/indicatorsets/utils.py index 921d4cb..5dda8c0 100644 --- a/src/indicatorsets/utils.py +++ b/src/indicatorsets/utils.py @@ -1,19 +1,18 @@ import ast +import json import random from collections import defaultdict from datetime import datetime as dtime from textwrap import dedent -import json import requests +from delphi_utils import get_structured_logger from django.conf import settings from django.http import JsonResponse from epiweeks import Week -from delphi_utils import get_structured_logger from indicatorsets.models import IndicatorSet - FLUVIEW_INDICATORS_MAPPING = {"wili": "%wILI", "ili": "%ILI"} @@ -643,8 +642,7 @@ def get_client_ip(request): ) -def log_form_stats(request, form_mode): - data = json.loads(request.body) +def log_form_stats(data, form_mode): log_data = { "form_mode": form_mode, "num_of_indicators": len(data.get("indicators", [])), @@ -662,17 +660,16 @@ def log_form_stats(request, form_mode): ), "api_key_used": bool(data.get("api_key")), "api_key": data.get("api_key", "")[:4] + "..." if data.get("api_key") else "", - "user_ip": get_client_ip(request), } get_structured_logger("form_stats").info(log_data) def log_form_data(request, form_mode): + data = json.loads(request.body) - log_form_stats(request, form_mode) + log_form_stats(data, form_mode) - data = json.loads(request.body) indicators = data.get("indicators", []) indicators = [ { From cf1dc9fa2c2d17df9d3a45057d9d631b5fc4e3c2 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 23 Sep 2025 17:54:02 +0300 Subject: [PATCH 44/49] Fixed api key logging --- src/indicatorsets/utils.py | 4 ++-- src/indicatorsets/views.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/indicatorsets/utils.py b/src/indicatorsets/utils.py index 5dda8c0..46b480e 100644 --- a/src/indicatorsets/utils.py +++ b/src/indicatorsets/utils.py @@ -730,8 +730,8 @@ def log_form_data(request, form_mode): "start_date": data.get("start_date", ""), "end_date": data.get("end_date", ""), "epiweeks": get_epiweek(data.get("start_date", ""), data.get("end_date", "")) if data.get("start_date") and data.get("end_date") else [], # fmt: skip - "api_key_used": bool(data.get("api_key")), - "api_key": data.get("api_key", "")[:4] + "..." if data.get("api_key") else "", + "api_key_used": bool(data.get("apiKey")), + "api_key": data.get("apiKey", "") if data.get("apiKey") else "", "user_ip": get_client_ip(request), } get_structured_logger("form_activity_logger").info(log_data) diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index c1e94ff..7be9904 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -252,6 +252,7 @@ def epivis(request): nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) flusurv_locations = data.get("flusurvLocations", []) + api_key = data.get("apiKey", None) log_form_data(request, "epivis") for indicator in indicators: if indicator["_endpoint"] == "covidcast": From d21329b4fccc7043805f23c98d0f312ecccbef15 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 23 Sep 2025 18:09:35 +0300 Subject: [PATCH 45/49] version bump --- src/epiportal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 5b3e2e4..fc0e811 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.8" +APP_VERSION = "1.0.9" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From 17ad0b9ae4b4ff2ce89e0a1d0ffba83b6caf4271 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Tue, 23 Sep 2025 22:02:19 +0300 Subject: [PATCH 46/49] Added clientID property --- src/assets/js/indicatorHandler.js | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 3b361a2..7954297 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -7,6 +7,29 @@ function dataLayerPush(payload) { } } + +function getGACookie() { + // Get all cookies as a string + const cookies = document.cookie.split(';'); + + // Find the _ga cookie + const gaCookie = cookies.find(cookie => cookie.trim().startsWith('_ga=')); + + if (gaCookie) { + // Extract the full _ga cookie value + const gaValue = gaCookie.split('=')[1]; + + // Extract the Client ID (remove the GA1.X prefix) + const clientId = gaValue.split('.').slice(2).join('.'); + + return clientId; + } + + return null; // Return null if _ga cookie is not found +} + +const clientId = getGACookie(); + class IndicatorHandler { constructor() { this.indicators = {}; @@ -366,7 +389,6 @@ class IndicatorHandler { const payload = { event: "submitSelectedIndicators", formMode: "epivis", - indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), @@ -374,6 +396,7 @@ class IndicatorHandler { flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), epivisUrl: data["epivis_url"], apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", + clientId: clientId ? clientId : "Not available", } dataLayerPush(payload); window.open(data["epivis_url"], '_blank').focus(); @@ -416,13 +439,13 @@ class IndicatorHandler { formMode: "export", formStartDate: submitData["start_date"], formEndDate: submitData["end_date"], - indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", + clientId: clientId ? clientId : "Not available", } dataLayerPush(payload); $('#modeSubmitResult').html(data["data_export_block"]); @@ -465,13 +488,13 @@ class IndicatorHandler { formMode: "preview", formStartDate: submitData["start_date"], formEndDate: submitData["end_date"], - indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", + clientId: clientId ? clientId : "Not available", } dataLayerPush(payload); $('#loader').hide(); @@ -523,13 +546,13 @@ class IndicatorHandler { formMode: "queryCode", formStartDate: submitData["start_date"], formEndDate: submitData["end_date"], - indicators: JSON.stringify(submitData["indicators"]), covidcastGeoValues: JSON.stringify(submitData["covidCastGeographicValues"]), fluviewGeoValues: JSON.stringify(submitData["fluviewLocations"]), nidssFluLocations: JSON.stringify(submitData["nidssFluLocations"]), nidssDengueLocations: JSON.stringify(submitData["nidssDengueLocations"]), flusurvLocations: JSON.stringify(submitData["flusurvLocations"]), apiKey: submitData["apiKey"] ? submitData["apiKey"] : "Not provided", + clientId: clientId ? clientId : "Not available", } dataLayerPush(payload); createQueryCodePython += data["python_code_blocks"].join("
"); From 31021cf1de24456ec9636448f966c7abc0729420 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Thu, 25 Sep 2025 23:36:42 +0300 Subject: [PATCH 47/49] Fixed logging --- src/assets/js/indicatorHandler.js | 7 ++- src/indicatorsets/utils.py | 86 ++++++++++++++++--------------- src/indicatorsets/views.py | 17 ++++-- 3 files changed, 64 insertions(+), 46 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 7954297..908576d 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -361,8 +361,11 @@ class IndicatorHandler { } plotData() { - const covidCastGeographicValues = - $("#geographic_value").select2("data"); + const covidCastGeographicValues = Object.groupBy( + $("#geographic_value").select2("data"), + ({ geoType }) => [geoType] + ); + console.log(covidCastGeographicValues); const fluviewLocations = $("#fluviewLocations").select2("data"); const nidssFluLocations = $("#nidssFluLocations").select2("data"); const nidssDengueLocations = $("#nidssDengueLocations").select2("data"); diff --git a/src/indicatorsets/utils.py b/src/indicatorsets/utils.py index 46b480e..e374559 100644 --- a/src/indicatorsets/utils.py +++ b/src/indicatorsets/utils.py @@ -1,12 +1,10 @@ import ast -import json import random from collections import defaultdict from datetime import datetime as dtime from textwrap import dedent import requests -from delphi_utils import get_structured_logger from django.conf import settings from django.http import JsonResponse from epiweeks import Week @@ -91,30 +89,31 @@ def group_by_property(list_of_dicts, property): def generate_covidcast_dataset_epivis(indicator, covidcast_geos): datasets = [] - for geo in covidcast_geos: - if geo["id"] not in indicator.get("notCoveredGeos", []): - geo_value = ( - geo["id"].split(":")[1].lower() - if geo["geoType"] in ["nation", "state"] - else geo["id"].split(":")[1] - ) - datasets.append( - { - "color": generate_random_color(), - "title": "value", - "params": { - "_endpoint": indicator["_endpoint"], - "data_source": indicator["data_source"], - "signal": indicator["indicator"], - "time_type": indicator["time_type"], - "geo_type": geo["geoType"], - "geo_value": geo_value, - "custom_title": generate_epivis_custom_title( - indicator, geo["text"] - ), - }, - } - ) + for geo_type in covidcast_geos.keys(): + for geo in covidcast_geos[geo_type]: + if geo["id"] not in indicator.get("notCoveredGeos", []): + geo_value = ( + geo["id"].split(":")[1].lower() + if geo["geoType"] in ["nation", "state"] + else geo["id"].split(":")[1] + ) + datasets.append( + { + "color": generate_random_color(), + "title": "value", + "params": { + "_endpoint": indicator["_endpoint"], + "data_source": indicator["data_source"], + "signal": indicator["indicator"], + "time_type": indicator["time_type"], + "geo_type": geo_type, + "geo_value": geo_value, + "custom_title": generate_epivis_custom_title( + indicator, geo["text"] + ), + }, + } + ) return datasets @@ -387,6 +386,7 @@ def preview_nidss_flu_data(nidss_flu_geos, start_date, end_date, api_key): "message": "API key does not exist. Register a new key at https://api.delphi.cmu.edu/epidata/admin/registration_form or contact delphi-support+privacy@andrew.cmu.edu to troubleshoot", } return JsonResponse(preview_data, safe=False) + return preview_data def preview_nidss_dengue_data(nidss_dengue_geos, start_date, end_date, api_key): @@ -642,7 +642,7 @@ def get_client_ip(request): ) -def log_form_stats(data, form_mode): +def log_form_stats(request, data, form_mode, logger): log_data = { "form_mode": form_mode, "num_of_indicators": len(data.get("indicators", [])), @@ -660,16 +660,14 @@ def log_form_stats(data, form_mode): ), "api_key_used": bool(data.get("api_key")), "api_key": data.get("api_key", "")[:4] + "..." if data.get("api_key") else "", + "user_ip": get_client_ip(request), + "user_ga_id": data.get("clientId", "") if data.get("clientId") else "", } - get_structured_logger("form_stats").info(log_data) + logger.info(log_data) -def log_form_data(request, form_mode): - data = json.loads(request.body) - - log_form_stats(data, form_mode) - +def log_form_data(request, data, form_mode, logger): indicators = data.get("indicators", []) indicators = [ { @@ -681,13 +679,18 @@ def log_form_data(request, form_mode): } for ind in indicators ] # fmt: skip indicators = group_by_property(indicators, "endpoint") - covidcast_geos = [ - { - "geo_type": geo.get("geoType"), - "geo_value": geo.get("id").split(":")[1], - "geo_text": geo.get("text"), - } for geo in data.get("covidCastGeographicValues", []) - ] # fmt: skip + covidcast_geographic_values = data.get("covidCastGeographicValues", []) + + covidcast_geos = [] + for geo_type in covidcast_geographic_values.keys(): + for geo_value in covidcast_geographic_values.get(geo_type, []): + covidcast_geos.append( + { + "geo_type": geo_type, + "geo_value": geo_value.get("id").split(":")[1], + "geo_text": geo_value.get("text"), + } + ) fluview_geos = [ { "geo_value": geo.get("id"), @@ -733,5 +736,6 @@ def log_form_data(request, form_mode): "api_key_used": bool(data.get("apiKey")), "api_key": data.get("apiKey", "") if data.get("apiKey") else "", "user_ip": get_client_ip(request), + "user_ga_id": data.get("clientId", "") if data.get("clientId") else "", } - get_structured_logger("form_activity_logger").info(log_data) + logger.info(log_data) diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index 7be9904..2f984d9 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -36,12 +36,16 @@ generate_query_code_nidss_dengue, generate_query_code_flusurv, log_form_data, + log_form_stats ) from delphi_utils import get_structured_logger indicatorsets_logger = get_structured_logger("indicatorsets_logger") +form_data_logger = get_structured_logger("form_data_logger") +form_stats_logger = get_structured_logger("form_stats_logger") + HEADER_DESCRIPTION = "Discover, display and download real-time infectious disease indicators (time series) that track a variety of pathogens, diseases and syndromes in a variety of locations (primarily within the USA). Browse the list, or filter it first by locations and pathogens of interest, by surveillance categories, and more. Expand any row to expose and select from a set of related indicators, then hit 'Show Selected Indicators' at bottom to plot or export your selected indicators, or to generate code snippets to retrieve them from the Delphi Epidata API. Most indicators are served from the Delphi Epidata real-time repository, but some may be available only from third parties or may require prior approval." @@ -252,8 +256,8 @@ def epivis(request): nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) flusurv_locations = data.get("flusurvLocations", []) - api_key = data.get("apiKey", None) - log_form_data(request, "epivis") + log_form_stats(request, data, "epivis", form_stats_logger) + log_form_data(request, data, "epivis", form_data_logger) for indicator in indicators: if indicator["_endpoint"] == "covidcast": datasets.extend( @@ -295,12 +299,15 @@ def generate_export_data_url(request): start_date = data.get("start_date", "") end_date = data.get("end_date", "") indicators = data.get("indicators", []) - covidcast_geos = data.get("covidCastGeographicValues", {}) + covidcast_geos = data.get("covidCastGeographicValues", []) fluview_geos = data.get("fluviewLocations", []) nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) flusurv_locations = data.get("flusurvLocations", []) api_key = data.get("apiKey", None) + + log_form_stats(request, data, "export", form_stats_logger) + log_form_data(request, data, "export", form_data_logger) data_export_commands.extend( generate_covidcast_indicators_export_url( indicators, start_date, end_date, covidcast_geos, api_key @@ -341,6 +348,8 @@ def generate_export_data_url(request): def preview_data(request): if request.method == "POST": data = json.loads(request.body) + log_form_stats(request, data, "preview", form_stats_logger) + log_form_data(request, data, "preview", form_data_logger) start_date = data.get("start_date", "") end_date = data.get("end_date", "") indicators = data.get("indicators", []) @@ -383,6 +392,8 @@ def preview_data(request): def create_query_code(request): if request.method == "POST": data = json.loads(request.body) + log_form_stats(request, data, "code", form_stats_logger) + log_form_data(request, data, "code", form_data_logger) start_date = data.get("start_date", "") end_date = data.get("end_date", "") indicators = data.get("indicators", []) From 69a1ffeeeea02fb54672ae96b39612b2fd9492b2 Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Fri, 26 Sep 2025 16:09:27 +0300 Subject: [PATCH 48/49] Version bump --- src/epiportal/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index fc0e811..440983e 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -24,7 +24,7 @@ from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration -APP_VERSION = "1.0.9" +APP_VERSION = "1.0.10" EPIVIS_URL = os.environ.get("EPIVIS_URL", "https://delphi.cmu.edu/epivis/") From 0e93717d5229d1e8a666e5b58e2e946934992dea Mon Sep 17 00:00:00 2001 From: Dmytro Trotsko Date: Mon, 29 Sep 2025 22:09:38 +0300 Subject: [PATCH 49/49] Fixed logging --- src/assets/js/indicatorHandler.js | 5 ++- src/epiportal/settings.py | 35 +++++++++++++++++++++ src/indicatorsets/utils.py | 52 +++++++++++++++++++++++-------- src/indicatorsets/views.py | 20 ++++++------ 4 files changed, 87 insertions(+), 25 deletions(-) diff --git a/src/assets/js/indicatorHandler.js b/src/assets/js/indicatorHandler.js index 908576d..4dbfdfb 100644 --- a/src/assets/js/indicatorHandler.js +++ b/src/assets/js/indicatorHandler.js @@ -365,7 +365,6 @@ class IndicatorHandler { $("#geographic_value").select2("data"), ({ geoType }) => [geoType] ); - console.log(covidCastGeographicValues); const fluviewLocations = $("#fluviewLocations").select2("data"); const nidssFluLocations = $("#nidssFluLocations").select2("data"); const nidssDengueLocations = $("#nidssDengueLocations").select2("data"); @@ -378,6 +377,7 @@ class IndicatorHandler { nidssDengueLocations: nidssDengueLocations, flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, + clientId: clientId, }; const csrftoken = Cookies.get("csrftoken"); $.ajax({ @@ -426,6 +426,7 @@ class IndicatorHandler { nidssDengueLocations: nidssDengueLocations, flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, + clientId: clientId, } const csrftoken = Cookies.get("csrftoken"); $.ajax({ @@ -476,6 +477,7 @@ class IndicatorHandler { nidssDengueLocations: nidssDengueLocations, flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, + clientId: clientId, } const csrftoken = Cookies.get("csrftoken"); $.ajax({ @@ -526,6 +528,7 @@ class IndicatorHandler { nidssDengueLocations: nidssDengueLocations, flusurvLocations: flusurvLocations, apiKey: document.getElementById("apiKey").value, + clientId: clientId, } const csrftoken = Cookies.get("csrftoken"); var createQueryCodePython = `

PYTHON PACKAGE

` diff --git a/src/epiportal/settings.py b/src/epiportal/settings.py index 440983e..5c56269 100644 --- a/src/epiportal/settings.py +++ b/src/epiportal/settings.py @@ -342,3 +342,38 @@ # django docs # https://django-docs.readthedocs.io/en/latest/ DOCS_ROOT = os.path.join(BASE_DIR, 'docs', 'build', 'html') + + +""" +REVERSE_PROXY_DEPTH is an integer value that indicates how many "chained" and trusted reverse proxies (like nginx) this + server instance is running behind. "chained" refers to proxies forwarding to other proxies, and then ultimately + forwarding to the app server itself. each of these proxies appends the remote address of the request to the + "X-Forwarded-For" header. in many situations, the most externally facing proxy (the first in the chain, the one that + faces the "open internet") can and should be set to write its own "X-Forwarded-For" header, ignoring and replacing + (or creating anew, if it didnt exist) such a header from the client request -- thus preserving the chain of trusted + proxies under our control. + +however, in our typical production environment, the most externally facing "proxy" is the AWS application load balancer, + which seemingly cannot be configured to provide this trust boundary without losing the referring client address + (see: https://docs.aws.amazon.com/elasticloadbalancing/latest/application/x-forwarded-headers.html ). accordingly, in + our current typical production environment, REVERSE_PROXY_DEPTH should be set to "2": one for the AWS application load + balancer, and one for our own nginx/haproxy instance. thus "2" is our default value. + +it is important that REVERSE_PROXY_DEPTH is set accurately for two reasons... + +setting it too high (or to -1) will respect more of the entries in the "X-Forwarded-For" header than are appropriate. + this can allow remote addresses to be "spoofed" when a client fakes this header, carrying security/identity + implications. in dev and testing, it is not particularly dangerous for this variable to be set to -1 (special case + for an "infinite" depth, where any and all proxy hops will be trusted). + +setting it too low can hinder logging accuracy -- that can cause an intermediate proxy IP address to be used as the + "real" client IP address, which could cause requests to be rate-limited inappropriately. + +setting REVERSE_PROXY_DEPTH to "0" essentially indicates there are no proxies between this server and the outside + world. in this case, the "X-Forwarded-For" header is ignored. +""" +REVERSE_PROXY_DEPTH = int(os.environ.get("PROXY_DEPTH", 4)) +# TODO: ^ this value should be "4" for the prod CC API server processes, and is currently unclear +# for prod AWS API server processes (but should be the same or lower)... when thats properly +# determined, set the default to the minimum of the two environments and special case the +# other in conf file(s). diff --git a/src/indicatorsets/utils.py b/src/indicatorsets/utils.py index e374559..32866a8 100644 --- a/src/indicatorsets/utils.py +++ b/src/indicatorsets/utils.py @@ -8,11 +8,15 @@ from django.conf import settings from django.http import JsonResponse from epiweeks import Week +from delphi_utils import get_structured_logger from indicatorsets.models import IndicatorSet FLUVIEW_INDICATORS_MAPPING = {"wili": "%wILI", "ili": "%ILI"} +form_data_logger = get_structured_logger("form_data_logger") +form_stats_logger = get_structured_logger("form_stats_logger") + def list_to_dict(lst): result = {} @@ -633,16 +637,37 @@ def generate_query_code_flusurv(flusurv_geos, start_date, end_date): return python_code_blocks, r_code_blocks -def get_client_ip(request): - x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") - return ( - x_forwarded_for.split(",")[0] - if x_forwarded_for - else request.META.get("REMOTE_ADDR") - ) +def get_real_ip_addr(req): # `req` should be a Flask.request object + if settings.REVERSE_PROXY_DEPTH: + # we only expect/trust (up to) "REVERSE_PROXY_DEPTH" number of proxies between this server and the outside world. + # a REVERSE_PROXY_DEPTH of 0 means not proxied, i.e. server is globally directly reachable. + # a negative proxy depth is a special case to trust the whole chain -- not generally recommended unless the + # most-external proxy is configured to disregard "X-Forwarded-For" from outside. + # really, ONLY trust the following headers if reverse proxied!!! + x_forwarded_for = req.META.get('HTTP_X_FORWARDED_FOR') + + if x_forwarded_for: + full_proxy_chain = x_forwarded_for.split(",") + # eliminate any extra addresses at the front of this list, as they could be spoofed. + if settings.REVERSE_PROXY_DEPTH > 0: + depth = settings.REVERSE_PROXY_DEPTH + else: + # special case for -1/negative: setting `depth` to 0 will not strip any items from the chain + depth = 0 + trusted_proxy_chain = full_proxy_chain[-depth:] + # accept the first (or only) address in the remaining trusted part of the chain as the actual remote address + return trusted_proxy_chain[0].strip() + + # fall back to "X-Real-Ip" if "X-Forwarded-For" isnt present + x_real_ip = req.META.get('HTTP_X_REAL_IP') + if x_real_ip: + return x_real_ip + + # if we are not proxied (or we are proxied but the headers werent present and we fell through to here), just use the remote ip addr as the true client address + return req.META.get('REMOTE_ADDR') -def log_form_stats(request, data, form_mode, logger): +def log_form_stats(request, data, form_mode): log_data = { "form_mode": form_mode, "num_of_indicators": len(data.get("indicators", [])), @@ -660,14 +685,15 @@ def log_form_stats(request, data, form_mode, logger): ), "api_key_used": bool(data.get("api_key")), "api_key": data.get("api_key", "")[:4] + "..." if data.get("api_key") else "", - "user_ip": get_client_ip(request), + "user_ip": get_real_ip_addr(request), "user_ga_id": data.get("clientId", "") if data.get("clientId") else "", } + form_stats_logger.info("form_stats", clientId=data.get("clientId", "")) - logger.info(log_data) + form_stats_logger.info("form_stats", **log_data) -def log_form_data(request, data, form_mode, logger): +def log_form_data(request, data, form_mode): indicators = data.get("indicators", []) indicators = [ { @@ -735,7 +761,7 @@ def log_form_data(request, data, form_mode, logger): "epiweeks": get_epiweek(data.get("start_date", ""), data.get("end_date", "")) if data.get("start_date") and data.get("end_date") else [], # fmt: skip "api_key_used": bool(data.get("apiKey")), "api_key": data.get("apiKey", "") if data.get("apiKey") else "", - "user_ip": get_client_ip(request), + "user_ip": get_real_ip_addr(request), "user_ga_id": data.get("clientId", "") if data.get("clientId") else "", } - logger.info(log_data) + form_data_logger.info("form_data", **log_data) diff --git a/src/indicatorsets/views.py b/src/indicatorsets/views.py index 2f984d9..6c7821d 100644 --- a/src/indicatorsets/views.py +++ b/src/indicatorsets/views.py @@ -36,15 +36,13 @@ generate_query_code_nidss_dengue, generate_query_code_flusurv, log_form_data, - log_form_stats + log_form_stats, ) from delphi_utils import get_structured_logger indicatorsets_logger = get_structured_logger("indicatorsets_logger") -form_data_logger = get_structured_logger("form_data_logger") -form_stats_logger = get_structured_logger("form_stats_logger") HEADER_DESCRIPTION = "Discover, display and download real-time infectious disease indicators (time series) that track a variety of pathogens, diseases and syndromes in a variety of locations (primarily within the USA). Browse the list, or filter it first by locations and pathogens of interest, by surveillance categories, and more. Expand any row to expose and select from a set of related indicators, then hit 'Show Selected Indicators' at bottom to plot or export your selected indicators, or to generate code snippets to retrieve them from the Delphi Epidata API. Most indicators are served from the Delphi Epidata real-time repository, but some may be available only from third parties or may require prior approval." @@ -256,8 +254,8 @@ def epivis(request): nidss_flu_locations = data.get("nidssFluLocations", []) nidss_dengue_locations = data.get("nidssDengueLocations", []) flusurv_locations = data.get("flusurvLocations", []) - log_form_stats(request, data, "epivis", form_stats_logger) - log_form_data(request, data, "epivis", form_data_logger) + log_form_stats(request, data, "epivis") + log_form_data(request, data, "epivis") for indicator in indicators: if indicator["_endpoint"] == "covidcast": datasets.extend( @@ -306,8 +304,8 @@ def generate_export_data_url(request): flusurv_locations = data.get("flusurvLocations", []) api_key = data.get("apiKey", None) - log_form_stats(request, data, "export", form_stats_logger) - log_form_data(request, data, "export", form_data_logger) + log_form_stats(request, data, "export") + log_form_data(request, data, "export") data_export_commands.extend( generate_covidcast_indicators_export_url( indicators, start_date, end_date, covidcast_geos, api_key @@ -348,8 +346,8 @@ def generate_export_data_url(request): def preview_data(request): if request.method == "POST": data = json.loads(request.body) - log_form_stats(request, data, "preview", form_stats_logger) - log_form_data(request, data, "preview", form_data_logger) + log_form_stats(request, data, "preview") + log_form_data(request, data, "preview") start_date = data.get("start_date", "") end_date = data.get("end_date", "") indicators = data.get("indicators", []) @@ -392,8 +390,8 @@ def preview_data(request): def create_query_code(request): if request.method == "POST": data = json.loads(request.body) - log_form_stats(request, data, "code", form_stats_logger) - log_form_data(request, data, "code", form_data_logger) + log_form_stats(request, data, "code") + log_form_data(request, data, "code") start_date = data.get("start_date", "") end_date = data.get("end_date", "") indicators = data.get("indicators", [])
{% for pathogen in indicator_set.pathogens.all %} - {{ pathogen }} + {{ pathogen }} {% endfor %} {{ indicator_set.geographic_scope }} {% for geo_level in indicator_set.get_geographic_levels %} - {{ geo_level }} + {{ geo_level }} {% endfor %} {{ indicator_set.temporal_scope_start }} {{ indicator_set.temporal_scope_end }} {% for item in indicator_set.temporal_granularity|split:"," %} - {{ item }} + {{ item }} {% endfor %} {{ indicator_set.reporting_cadence }}{{ indicator_set.demographic_granularity }} {% for severity_pyramid_rung in indicator_set.severity_pyramid_rungs.all %} - {{ severity_pyramid_rung }} + {{ severity_pyramid_rung }} {% endfor %} {{ indicator_set.original_data_provider }}{{ indicator_set.geographic_scope }} - {% for geo_level in indicator_set.get_geographic_levels %} - {{ geo_level }} + {% for geo_level in indicator_set.geographic_levels.all %} + {{ geo_level.display_name }} {% endfor %} {{ indicator_set.temporal_scope_start }}