From a17dee865b2dcd391ce17181341bcd465cee16a3 Mon Sep 17 00:00:00 2001 From: chinandrew Date: Mon, 3 Jan 2022 11:38:00 -0800 Subject: [PATCH 1/8] Add make install-ci target to install without editabel mode --- .github/workflows/python-ci.yml | 2 +- _delphi_utils_python/Makefile | 5 +++++ _template_python/Makefile | 6 ++++++ cdc_covidnet/Makefile | 6 ++++++ changehc/Makefile | 6 ++++++ claims_hosp/Makefile | 6 ++++++ combo_cases_and_deaths/Makefile | 6 ++++++ covid_act_now/Makefile | 6 ++++++ doctor_visits/Makefile | 6 ++++++ google_symptoms/Makefile | 6 ++++++ hhs_facilities/Makefile | 6 ++++++ hhs_hosp/Makefile | 6 ++++++ jhu/Makefile | 6 ++++++ nchs_mortality/Makefile | 6 ++++++ nowcast/Makefile | 6 ++++++ quidel/Makefile | 6 ++++++ quidel_covidtest/Makefile | 6 ++++++ safegraph_patterns/Makefile | 6 ++++++ sir_complainsalot/Makefile | 6 ++++++ usafacts/Makefile | 6 ++++++ 20 files changed, 114 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index adeb011a6..47c6b2f16 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -32,7 +32,7 @@ jobs: pip install pylint pytest pydocstyle wheel - name: Install run: | - make install + make install-ci - name: Lint run: | make lint diff --git a/_delphi_utils_python/Makefile b/_delphi_utils_python/Makefile index f301790c4..dd9c5f37f 100644 --- a/_delphi_utils_python/Makefile +++ b/_delphi_utils_python/Makefile @@ -8,6 +8,11 @@ install: venv pip install wheel ; \ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install . + lint: . env/bin/activate; pylint delphi_utils . env/bin/activate; pydocstyle delphi_utils diff --git a/_template_python/Makefile b/_template_python/Makefile index 72814c37b..bdea33afd 100644 --- a/_template_python/Makefile +++ b/_template_python/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/cdc_covidnet/Makefile b/cdc_covidnet/Makefile index 72814c37b..bdea33afd 100644 --- a/cdc_covidnet/Makefile +++ b/cdc_covidnet/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/changehc/Makefile b/changehc/Makefile index 72814c37b..bdea33afd 100644 --- a/changehc/Makefile +++ b/changehc/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/claims_hosp/Makefile b/claims_hosp/Makefile index 72814c37b..bdea33afd 100644 --- a/claims_hosp/Makefile +++ b/claims_hosp/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/combo_cases_and_deaths/Makefile b/combo_cases_and_deaths/Makefile index 72814c37b..bdea33afd 100644 --- a/combo_cases_and_deaths/Makefile +++ b/combo_cases_and_deaths/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/covid_act_now/Makefile b/covid_act_now/Makefile index 72814c37b..bdea33afd 100644 --- a/covid_act_now/Makefile +++ b/covid_act_now/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/doctor_visits/Makefile b/doctor_visits/Makefile index 72814c37b..bdea33afd 100644 --- a/doctor_visits/Makefile +++ b/doctor_visits/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/google_symptoms/Makefile b/google_symptoms/Makefile index a58351578..31b50d062 100644 --- a/google_symptoms/Makefile +++ b/google_symptoms/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/hhs_facilities/Makefile b/hhs_facilities/Makefile index 72814c37b..bdea33afd 100644 --- a/hhs_facilities/Makefile +++ b/hhs_facilities/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/hhs_hosp/Makefile b/hhs_hosp/Makefile index e0aca95c0..210b6b693 100644 --- a/hhs_hosp/Makefile +++ b/hhs_hosp/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/jhu/Makefile b/jhu/Makefile index 72814c37b..bdea33afd 100644 --- a/jhu/Makefile +++ b/jhu/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/nchs_mortality/Makefile b/nchs_mortality/Makefile index 72814c37b..bdea33afd 100644 --- a/nchs_mortality/Makefile +++ b/nchs_mortality/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/nowcast/Makefile b/nowcast/Makefile index 519f2382e..3a38afc36 100644 --- a/nowcast/Makefile +++ b/nowcast/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) --match-dir '(?!nowcast_fusion)' diff --git a/quidel/Makefile b/quidel/Makefile index 72814c37b..bdea33afd 100644 --- a/quidel/Makefile +++ b/quidel/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/quidel_covidtest/Makefile b/quidel_covidtest/Makefile index 72814c37b..bdea33afd 100644 --- a/quidel_covidtest/Makefile +++ b/quidel_covidtest/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/safegraph_patterns/Makefile b/safegraph_patterns/Makefile index 72814c37b..bdea33afd 100644 --- a/safegraph_patterns/Makefile +++ b/safegraph_patterns/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/sir_complainsalot/Makefile b/sir_complainsalot/Makefile index 72814c37b..bdea33afd 100644 --- a/sir_complainsalot/Makefile +++ b/sir_complainsalot/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) diff --git a/usafacts/Makefile b/usafacts/Makefile index 72814c37b..bdea33afd 100644 --- a/usafacts/Makefile +++ b/usafacts/Makefile @@ -11,6 +11,12 @@ install: venv pip install -e ../_delphi_utils_python ;\ pip install -e . +install-ci: venv + . env/bin/activate; \ + pip install wheel ; \ + pip install ../_delphi_utils_python ;\ + pip install . + lint: . env/bin/activate; pylint $(dir) . env/bin/activate; pydocstyle $(dir) From 8db1d50cb73913dfcd362f8700880ad7b70d1a31 Mon Sep 17 00:00:00 2001 From: chinandrew Date: Mon, 3 Jan 2022 12:18:51 -0800 Subject: [PATCH 2/8] Fix nowcast directory by adding inits to subdirs --- nowcast/Makefile | 2 +- nowcast/delphi_nowcast/deconvolution/__init__.py | 0 nowcast/delphi_nowcast/nowcast_fusion/__init__.py | 0 nowcast/delphi_nowcast/sensorization/__init__.py | 0 4 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 nowcast/delphi_nowcast/deconvolution/__init__.py create mode 100644 nowcast/delphi_nowcast/nowcast_fusion/__init__.py create mode 100644 nowcast/delphi_nowcast/sensorization/__init__.py diff --git a/nowcast/Makefile b/nowcast/Makefile index 3a38afc36..a59dda820 100644 --- a/nowcast/Makefile +++ b/nowcast/Makefile @@ -1,6 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]* | head -1') venv: python3.8 -m venv env diff --git a/nowcast/delphi_nowcast/deconvolution/__init__.py b/nowcast/delphi_nowcast/deconvolution/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nowcast/delphi_nowcast/nowcast_fusion/__init__.py b/nowcast/delphi_nowcast/nowcast_fusion/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nowcast/delphi_nowcast/sensorization/__init__.py b/nowcast/delphi_nowcast/sensorization/__init__.py new file mode 100644 index 000000000..e69de29bb From e822782271458f5ea3813906115332c46aa68c13 Mon Sep 17 00:00:00 2001 From: chinandrew Date: Mon, 3 Jan 2022 12:19:29 -0800 Subject: [PATCH 3/8] Limit directory find to 1 result so subdirectories dont cause issues --- _template_python/Makefile | 3 +-- cdc_covidnet/Makefile | 3 +-- changehc/Makefile | 3 +-- claims_hosp/Makefile | 3 +-- combo_cases_and_deaths/Makefile | 3 +-- covid_act_now/Makefile | 3 +-- doctor_visits/Makefile | 3 +-- google_symptoms/Makefile | 3 +-- hhs_facilities/Makefile | 3 +-- hhs_hosp/Makefile | 3 +-- jhu/Makefile | 3 +-- nchs_mortality/Makefile | 3 +-- nowcast/Makefile | 3 +-- quidel/Makefile | 3 +-- quidel_covidtest/Makefile | 3 +-- safegraph_patterns/Makefile | 3 +-- sir_complainsalot/Makefile | 3 +-- usafacts/Makefile | 3 +-- 18 files changed, 18 insertions(+), 36 deletions(-) diff --git a/_template_python/Makefile b/_template_python/Makefile index bdea33afd..bc88f1fec 100644 --- a/_template_python/Makefile +++ b/_template_python/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/cdc_covidnet/Makefile b/cdc_covidnet/Makefile index bdea33afd..bc88f1fec 100644 --- a/cdc_covidnet/Makefile +++ b/cdc_covidnet/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/changehc/Makefile b/changehc/Makefile index bdea33afd..bc88f1fec 100644 --- a/changehc/Makefile +++ b/changehc/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/claims_hosp/Makefile b/claims_hosp/Makefile index bdea33afd..bc88f1fec 100644 --- a/claims_hosp/Makefile +++ b/claims_hosp/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/combo_cases_and_deaths/Makefile b/combo_cases_and_deaths/Makefile index bdea33afd..bc88f1fec 100644 --- a/combo_cases_and_deaths/Makefile +++ b/combo_cases_and_deaths/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/covid_act_now/Makefile b/covid_act_now/Makefile index bdea33afd..bc88f1fec 100644 --- a/covid_act_now/Makefile +++ b/covid_act_now/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/doctor_visits/Makefile b/doctor_visits/Makefile index bdea33afd..bc88f1fec 100644 --- a/doctor_visits/Makefile +++ b/doctor_visits/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/google_symptoms/Makefile b/google_symptoms/Makefile index 31b50d062..f6a5b7e63 100644 --- a/google_symptoms/Makefile +++ b/google_symptoms/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/hhs_facilities/Makefile b/hhs_facilities/Makefile index bdea33afd..bc88f1fec 100644 --- a/hhs_facilities/Makefile +++ b/hhs_facilities/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/hhs_hosp/Makefile b/hhs_hosp/Makefile index 210b6b693..ea591dcb5 100644 --- a/hhs_hosp/Makefile +++ b/hhs_hosp/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/jhu/Makefile b/jhu/Makefile index bdea33afd..bc88f1fec 100644 --- a/jhu/Makefile +++ b/jhu/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/nchs_mortality/Makefile b/nchs_mortality/Makefile index bdea33afd..bc88f1fec 100644 --- a/nchs_mortality/Makefile +++ b/nchs_mortality/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/nowcast/Makefile b/nowcast/Makefile index a59dda820..38c796e92 100644 --- a/nowcast/Makefile +++ b/nowcast/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]* | head -1') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/quidel/Makefile b/quidel/Makefile index bdea33afd..bc88f1fec 100644 --- a/quidel/Makefile +++ b/quidel/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/quidel_covidtest/Makefile b/quidel_covidtest/Makefile index bdea33afd..bc88f1fec 100644 --- a/quidel_covidtest/Makefile +++ b/quidel_covidtest/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/safegraph_patterns/Makefile b/safegraph_patterns/Makefile index bdea33afd..bc88f1fec 100644 --- a/safegraph_patterns/Makefile +++ b/safegraph_patterns/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/sir_complainsalot/Makefile b/sir_complainsalot/Makefile index bdea33afd..bc88f1fec 100644 --- a/sir_complainsalot/Makefile +++ b/sir_complainsalot/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env diff --git a/usafacts/Makefile b/usafacts/Makefile index bdea33afd..bc88f1fec 100644 --- a/usafacts/Makefile +++ b/usafacts/Makefile @@ -1,7 +1,6 @@ .PHONY = venv, lint, test, clean -dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*') - +dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: python3.8 -m venv env From 3748141d22e114f2d0a773840f88cb87341f076e Mon Sep 17 00:00:00 2001 From: chinandrew Date: Mon, 3 Jan 2022 15:23:38 -0800 Subject: [PATCH 4/8] Fix nowcast linting --- nowcast/.pylintrc | 6 +-- .../deconvolution/deconvolution.py | 2 +- .../deconvolution/ground_truth.py | 47 ------------------- .../delphi_nowcast/sensorization/ar_model.py | 4 +- .../sensorization/regression_model.py | 4 +- .../delphi_nowcast/sensorization/sensor.py | 5 +- 6 files changed, 12 insertions(+), 56 deletions(-) delete mode 100644 nowcast/delphi_nowcast/deconvolution/ground_truth.py diff --git a/nowcast/.pylintrc b/nowcast/.pylintrc index 90d5d3f91..786c72eb8 100644 --- a/nowcast/.pylintrc +++ b/nowcast/.pylintrc @@ -16,9 +16,9 @@ disable=logging-format-interpolation, [BASIC] # Allow arbitrarily short-named variables. -variable-rgx=[a-z_][a-z0-9_]* -argument-rgx=[a-z_][a-z0-9_]* -attr-rgx=[a-z_][a-z0-9_]* +variable-rgx=[A-Za-z_][a-z0-9_]* +argument-rgx=[A-Za-z_][a-z0-9_]* +attr-rgx=[A-Za-z_][a-z0-9_]* [DESIGN] diff --git a/nowcast/delphi_nowcast/deconvolution/deconvolution.py b/nowcast/delphi_nowcast/deconvolution/deconvolution.py index dd6b00f61..03e4a9c08 100644 --- a/nowcast/delphi_nowcast/deconvolution/deconvolution.py +++ b/nowcast/delphi_nowcast/deconvolution/deconvolution.py @@ -74,7 +74,7 @@ def deconvolve_double_smooth_ntf( x_k = None alpha_0 = np.zeros(n - k - 1) u_0 = np.zeros(n - k - 1) - for t in range(n_iters): + for _ in range(n_iters): x_k = x_update_1 @ (Cty + rho * D.T @ (alpha_0 + u_0)) Dx = D @ x_k alpha_k = _soft_thresh(Dx - u_0, lam / rho) diff --git a/nowcast/delphi_nowcast/deconvolution/ground_truth.py b/nowcast/delphi_nowcast/deconvolution/ground_truth.py deleted file mode 100644 index f251771f7..000000000 --- a/nowcast/delphi_nowcast/deconvolution/ground_truth.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Functions to construct the ground truth for a given location.""" -from datetime import date -from typing import List, Dict, Tuple - -import numpy as np - -from .deconvolution import deconvolve_double_smooth_tf_cv -from ..constants import Default -from ..data_containers import LocationSeries, SensorConfig -from ..epidata import get_historical_sensor_data, get_indicator_data, export_to_csv - - -def construct_truths(start_date: date, - end_date: date, - as_of: date, # most likely today - truth: SensorConfig, - locations: List[LocationSeries], - export_dir: str = "") -> Dict[Tuple, LocationSeries]: - raw_indicator = get_indicator_data([truth], locations, as_of) - output = {} - for location in locations: - indicator_key = (truth.source, truth.signal, location.geo_type, location.geo_value) - location, missing_dates = get_historical_sensor_data(truth, location, end_date, start_date) - location, export = fill_missing_days(location, raw_indicator[indicator_key], missing_dates) - output[indicator_key] = location - if export_dir and export.values: - export_to_csv(export, truth, as_of, export_dir) - return output - - -def fill_missing_days(stored_vals: LocationSeries, - indicator_data: LocationSeries, - missing_dates: List[date]): - export_data = LocationSeries(stored_vals.geo_value, stored_vals.geo_type) - for day in missing_dates: - try: - y = np.array(indicator_data.get_data_range(min(indicator_data.dates), day, "linear")) - x = np.arange(1, len(y) + 1) - except ValueError: - continue - deconv_vals = deconvolve_double_smooth_tf_cv( - y, x, Default.DELAY_DISTRIBUTION - ) - missing_day_val = deconv_vals[-1] - stored_vals.add_data(day, missing_day_val) - export_data.add_data(day, missing_day_val) # holds only data to get exported - return stored_vals, export_data diff --git a/nowcast/delphi_nowcast/sensorization/ar_model.py b/nowcast/delphi_nowcast/sensorization/ar_model.py index ca7bc68db..34feb9b74 100644 --- a/nowcast/delphi_nowcast/sensorization/ar_model.py +++ b/nowcast/delphi_nowcast/sensorization/ar_model.py @@ -1,4 +1,6 @@ -from datetime import datetime, timedelta, date +"""Fit autoregression models.""" + +from datetime import timedelta, date from typing import Tuple import numpy as np diff --git a/nowcast/delphi_nowcast/sensorization/regression_model.py b/nowcast/delphi_nowcast/sensorization/regression_model.py index dab70d0ba..a22cc01c3 100644 --- a/nowcast/delphi_nowcast/sensorization/regression_model.py +++ b/nowcast/delphi_nowcast/sensorization/regression_model.py @@ -1,4 +1,6 @@ -from datetime import datetime, timedelta, date +"""Fit linear regression mdels.""" + +from datetime import timedelta, date import numpy as np diff --git a/nowcast/delphi_nowcast/sensorization/sensor.py b/nowcast/delphi_nowcast/sensorization/sensor.py index 913e07fce..39fa07876 100644 --- a/nowcast/delphi_nowcast/sensorization/sensor.py +++ b/nowcast/delphi_nowcast/sensorization/sensor.py @@ -1,5 +1,4 @@ """Functions to run sensorization.""" -import os from collections import defaultdict from typing import List, DefaultDict from datetime import timedelta, date @@ -54,7 +53,7 @@ def compute_sensors(as_of_date: date, (sensor.source, sensor.signal, loc.geo_type, loc.geo_value) ) if not covariates: - # TODO convert to log statements #689 + # TODO convert to log statements #689 # pylint: disable=fixme print(f"No data: {(sensor.source, sensor.signal, loc.geo_type, loc.geo_value)}") continue reg_sensor = compute_regression_sensor(sensor_pred_date, covariates, loc, REG_INTERCEPT) @@ -95,7 +94,7 @@ def historical_sensors(start_date: date, output = defaultdict(list) for location in ground_truths: for sensor in sensors: - sensor_vals, missing_dates = get_historical_sensor_data( + sensor_vals, _ = get_historical_sensor_data( sensor, location, start_date, end_date ) if sensor_vals.data: From 02b3aa05a049f70686dfc95e0d7160e9613f660d Mon Sep 17 00:00:00 2001 From: Katie Mazaitis Date: Thu, 6 Jan 2022 10:12:29 -0500 Subject: [PATCH 5/8] [chng] Update expected file count We now get an additional file for flu inpatient. --- changehc/delphi_changehc/download_ftp_files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changehc/delphi_changehc/download_ftp_files.py b/changehc/delphi_changehc/download_ftp_files.py index 8a1cb1d67..46846102b 100644 --- a/changehc/delphi_changehc/download_ftp_files.py +++ b/changehc/delphi_changehc/download_ftp_files.py @@ -32,8 +32,8 @@ def get_files_from_dir(sftp, filedate, out_path): not path.exists(path.join(out_path, filename)): filepaths_to_download[filename] = path.join(out_path, filename) - # make sure we don't download more than 6 files per day - assert len(filepaths_to_download) <= 6, "more files dropped than expected" + # make sure we don't download more than 7 files per day + assert len(filepaths_to_download) <= 7, "more files dropped than expected" # download! for infile, outfile in filepaths_to_download.items(): From cd05134afafb3ecd51b733c091ce476096e11df0 Mon Sep 17 00:00:00 2001 From: Kathryn M Mazaitis Date: Thu, 6 Jan 2022 12:52:33 -0500 Subject: [PATCH 6/8] Factor out constant and update tests --- changehc/delphi_changehc/constants.py | 2 ++ .../delphi_changehc/download_ftp_files.py | 5 +++-- changehc/tests/test_download_ftp_files.py | 21 ++++++++----------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/changehc/delphi_changehc/constants.py b/changehc/delphi_changehc/constants.py index 38ee1e2c1..c934c556a 100644 --- a/changehc/delphi_changehc/constants.py +++ b/changehc/delphi_changehc/constants.py @@ -9,3 +9,5 @@ NA = "NA" HRR = "hrr" FIPS = "fips" + +EXPECTED_FILES_PER_DROP = 7 diff --git a/changehc/delphi_changehc/download_ftp_files.py b/changehc/delphi_changehc/download_ftp_files.py index 46846102b..f85ef9944 100644 --- a/changehc/delphi_changehc/download_ftp_files.py +++ b/changehc/delphi_changehc/download_ftp_files.py @@ -8,6 +8,7 @@ # third party import paramiko +from .constants import EXPECTED_FILES_PER_DROP def print_callback(filename, bytes_so_far, bytes_total): """Log file transfer progress.""" @@ -32,8 +33,8 @@ def get_files_from_dir(sftp, filedate, out_path): not path.exists(path.join(out_path, filename)): filepaths_to_download[filename] = path.join(out_path, filename) - # make sure we don't download more than 7 files per day - assert len(filepaths_to_download) <= 7, "more files dropped than expected" + # make sure we don't download too many files per day + assert len(filepaths_to_download) <= EXPECTED_FILES_PER_DROP, "more files dropped than expected" # download! for infile, outfile in filepaths_to_download.items(): diff --git a/changehc/tests/test_download_ftp_files.py b/changehc/tests/test_download_ftp_files.py index 8e8dd45c1..0338e2cd4 100644 --- a/changehc/tests/test_download_ftp_files.py +++ b/changehc/tests/test_download_ftp_files.py @@ -6,6 +6,7 @@ # first party from delphi_changehc.download_ftp_files import * +from delphi_changehc.constants import EXPECTED_FILES_PER_DROP class TestDownloadFTPFiles: @@ -51,22 +52,18 @@ def test_get_files(self, mock_path): get_files_from_dir(one_new_one_old, "00005566", "") assert one_new_one_old.num_gets == 1 - # When seven new files are present, AssertionError - new_file1 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo1") - new_file2 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo2") - new_file3 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo3") - new_file4 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo4") - new_file5 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo5") - new_file6 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo6") - new_file7 = self.FileAttr(dt.timestamp(dt.now()), "00001122_foo7") - seven_new = self.MockSFTP([new_file1, new_file2, new_file3, new_file4, - new_file5, new_file6, new_file7]) + # When too many new files are present, AssertionError + file_batch = [ + self.FileAttr(dt.timestamp(dt.now()), f"00001122_foo{i}") + for i in range(EXPECTED_FILES_PER_DROP + 1) + ] + too_many_new = self.MockSFTP(file_batch) with pytest.raises(AssertionError): - get_files_from_dir(seven_new, "00001122", "") + get_files_from_dir(too_many_new, "00001122", "") # When the file already exists, no files are downloaded mock_path.exists.return_value = True - one_exists = self.MockSFTP([new_file1]) + one_exists = self.MockSFTP([file_batch[0]]) get_files_from_dir(one_new, "00001122", "") assert one_exists.num_gets == 0 From 0b2b3d41ccc8e68f7ee4bb116688cd9179748679 Mon Sep 17 00:00:00 2001 From: Delphi Deploy Bot Date: Thu, 6 Jan 2022 20:20:21 +0000 Subject: [PATCH 7/8] chore: bump delphi_utils to 0.2.8 --- _delphi_utils_python/.bumpversion.cfg | 2 +- _delphi_utils_python/delphi_utils/__init__.py | 2 +- _delphi_utils_python/setup.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/_delphi_utils_python/.bumpversion.cfg b/_delphi_utils_python/.bumpversion.cfg index d4ff51710..222ae37ed 100644 --- a/_delphi_utils_python/.bumpversion.cfg +++ b/_delphi_utils_python/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.7 +current_version = 0.2.8 commit = True message = chore: bump delphi_utils to {new_version} tag = False diff --git a/_delphi_utils_python/delphi_utils/__init__.py b/_delphi_utils_python/delphi_utils/__init__.py index 67331c00d..df6bb0402 100644 --- a/_delphi_utils_python/delphi_utils/__init__.py +++ b/_delphi_utils_python/delphi_utils/__init__.py @@ -15,4 +15,4 @@ from .nancodes import Nans from .weekday import Weekday -__version__ = "0.2.7" +__version__ = "0.2.8" diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py index a463140fa..53c9a4b1c 100644 --- a/_delphi_utils_python/setup.py +++ b/_delphi_utils_python/setup.py @@ -26,7 +26,7 @@ setup( name="delphi_utils", - version="0.2.7", + version="0.2.8", description="Shared Utility Functions for Indicators", long_description=long_description, long_description_content_type="text/markdown", From c219d892866a9c2b66098483595325a55618370a Mon Sep 17 00:00:00 2001 From: Delphi Deploy Bot Date: Thu, 6 Jan 2022 20:20:21 +0000 Subject: [PATCH 8/8] chore: bump covidcast-indicators to 0.2.19 --- .bumpversion.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index aa0ea322f..224acb5f4 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.2.18 +current_version = 0.2.19 commit = True message = chore: bump covidcast-indicators to {new_version} tag = False