Merge pull request #1653 from cmu-delphi/release/indicators_v0.3.17_utils_v0.3.5

Release covidcast-indicators 0.3.17

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.16
+current_version = 0.3.17
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

ansible/ansible-deploy-staging.yaml

@@ -27,6 +27,10 @@
       local_action: stat path="templates/{{ indicator }}-params-prod.json.j2"
       register: template
 
+    - name: Check to see if we have a secrets template to send.
+      local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2"
+      register: template_secrets
+
     - name: Set production params file.
       copy:
         src: files/{{ indicator }}-params-prod.json
@@ -42,3 +46,11 @@
         owner: "{{ runtime_user }}"
         group: "{{ runtime_user }}"
       when: template.stat.exists
+
+    - name: Set production secrets template.
+      template:
+        src: templates/{{ indicator }}-secrets-prod.py.j2
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+      when: template_secrets.stat.exists

ansible/ansible-deploy.yaml

@@ -27,6 +27,10 @@
       local_action: stat path="templates/{{ indicator }}-params-prod.json.j2"
       register: template
 
+    - name: Check to see if we have a secrets template to send.
+      local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2"
+      register: template_secrets
+
     - name: Set production params file.
       copy:
         src: files/{{ indicator }}-params-prod.json
@@ -42,3 +46,11 @@
         owner: "{{ runtime_user }}"
         group: "{{ runtime_user }}"
       when: template.stat.exists
+
+    - name: Set production secrets template.
+      template:
+        src: templates/{{ indicator }}-secrets-prod.py.j2
+        dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py"
+        owner: "{{ runtime_user }}"
+        group: "{{ runtime_user }}"
+      when: template_secrets.stat.exists

ansible/templates/claims_hosp-params-prod.py.j2

@@ -0,0 +1,43 @@
+{
+  "common": {
+    "export_dir": "./receiving",
+    "log_exceptions": false
+  },
+  "indicator": {
+    "input_dir": "./retrieve_files",
+    "start_date": "2020-02-01",
+    "end_date": null,
+    "drop_date": null,
+    "n_backfill_days": 70,
+    "n_waiting_days": 3,
+    "write_se": false,
+    "obfuscated_prefix": "foo_obfuscated",
+    "parallel": false,
+    "geos": ["state", "msa", "hrr", "county"],
+    "weekday": [true, false],
+    "ftp_credentials": {
+      "host": "{{ claims_hosp_ftp_host }}",
+      "user": "{{ claims_hosp_ftp_user }}",
+      "pass": "{{ claims_hosp_ftp_password }}",
+      "port": 2222
+    }
+  },
+  "validation": {
+    "common": {
+      "data_source": "hospital-admissions",
+      "span_length": 14,
+      "min_expected_lag": {"all": "3"},
+      "max_expected_lag": {"all": "4"},
+      "dry_run": true,
+      "suppressed_errors": []
+    },
+    "static": {
+      "minimum_sample_size": 5,
+      "missing_se_allowed": true,
+      "missing_sample_size_allowed": true
+    },
+    "dynamic": {
+      "ref_window_size": 7
+    }
+  }
+}

ansible/templates/claims_hosp-secrets-prod.py.j2

@@ -0,0 +1,11 @@
+class claims:
+    HOST = 'ftp.delphi.cmu.edu'
+    USER = {{ claims_hosp_ftp_user }}
+    PASS = {{ claims_hosp_ftp_password }}
+    PORT = 2222
+
+
+class covidcast:
+    HOST = "delphi.midas.cs.cmu.edu"
+    USER = {{ claims_hosp_midas_user }}
+    PASS = {{ claims_hosp_midas_password }}

ansible/templates/sir_complainsalot-params-prod.json.j2

@@ -43,100 +43,6 @@
       "maintainers": ["U01AP8GSWG3","U01069KCRS7"],
       "retired-signals": ["completely_home_prop", "full_time_work_prop", "part_time_work_prop", "median_home_dwell_time", "completely_home_prop_7dav", "full_time_work_prop_7dav", "part_time_work_prop_7dav", "median_home_dwell_time_7dav"]
     },
-    "fb-survey": {
-      "max_age": 3,
-      "maintainers": ["U01069KCRS7"],
-      "retired-signals": [
-        "smoothed_anxious_5d", "smoothed_wanxious_5d",
-        "smoothed_depressed_5d", "smoothed_wdepressed_5d",
-        "smoothed_felt_isolated_5d", "smoothed_wfelt_isolated_5d",
-        "smoothed_large_event_1d", "smoothed_wlarge_event_1d",
-        "smoothed_restaurant_1d", "smoothed_wrestaurant_1d",
-        "smoothed_shop_1d", "smoothed_wshop_1d",
-        "smoothed_spent_time_1d", "smoothed_wspent_time_1d",
-        "smoothed_travel_outside_state_5d", "smoothed_wtravel_outside_state_5d",
-        "smoothed_work_outside_home_1d", "smoothed_wwork_outside_home_1d",
-        "smoothed_wearing_mask", "smoothed_wwearing_mask",
-        "smoothed_vaccine_likely_local_health", "smoothed_wvaccine_likely_local_health",
-        "smoothed_others_masked", "smoothed_wothers_masked",
-        "smoothed_wanted_test_14d", "smoothed_wwanted_test_14d",
-        "smoothed_covid_vaccinated_or_accept", "smoothed_wcovid_vaccinated_or_accept",
-        "smoothed_accept_covid_vaccine", "smoothed_waccept_covid_vaccine",
-        "smoothed_hesitancy_reason_allergic", "smoothed_whesitancy_reason_allergic",
-        "smoothed_hesitancy_reason_not_recommended", "smoothed_whesitancy_reason_not_recommended",
-        "smoothed_hesitancy_reason_distrust_vaccines", "smoothed_whesitancy_reason_distrust_vaccines",
-        "smoothed_hesitancy_reason_health_condition", "smoothed_whesitancy_reason_health_condition",
-        "smoothed_hesitancy_reason_pregnant", "smoothed_whesitancy_reason_pregnant",
-        "smoothed_vaccine_likely_friends", "smoothed_wvaccine_likely_friends",
-        "smoothed_vaccine_likely_who", "smoothed_wvaccine_likely_who",
-        "smoothed_vaccine_likely_govt_health", "smoothed_wvaccine_likely_govt_health",
-        "smoothed_vaccine_likely_politicians", "smoothed_wvaccine_likely_politicians",
-        "smoothed_vaccine_likely_doctors", "smoothed_wvaccine_likely_doctors",
-        "smoothed_felt_isolated_7d", "smoothed_wfelt_isolated_7d",
-        "smoothed_worried_become_ill", "smoothed_wworried_become_ill",
-        "smoothed_received_2_vaccine_doses", "smoothed_wreceived_2_vaccine_doses",
-        "smoothed_hesitancy_reason_dislike_vaccines", "smoothed_whesitancy_reason_dislike_vaccines",
-        "smoothed_inperson_school_fulltime", "smoothed_winperson_school_fulltime",
-        "smoothed_inperson_school_parttime", "smoothed_winperson_school_parttime",
-        "smoothed_vaccinate_children", "smoothed_wvaccinate_children",
-        ["smoothed_vaccine_barrier_appointment_time_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_appointment_time_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_childcare_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_childcare_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_document_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_document_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_eligible_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_eligible_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_language_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_language_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_no_appointments_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_no_appointments_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_none_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_none_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_technical_difficulties_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_technical_difficulties_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_technology_access_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_technology_access_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_time_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_time_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_travel_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_travel_tried", "msa", "county", "state"],
-        ["smoothed_vaccine_barrier_type_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_type_tried", "msa", "county", "state"],
-        ["smoothed_try_vaccinate_1m", "hrr"], ["smoothed_wtry_vaccinate_1m", "hrr"],
-        ["smoothed_try_vaccinate_1m", "msa"], ["smoothed_wtry_vaccinate_1m", "msa"],
-        ["smoothed_dontneed_reason_dont_spend_time", "hrr"], ["smoothed_wdontneed_reason_dont_spend_time", "hrr", "msa"],
-        ["smoothed_dontneed_reason_had_covid", "hrr"], ["smoothed_wdontneed_reason_had_covid", "hrr", "msa"],
-        ["smoothed_dontneed_reason_not_beneficial", "hrr"], ["smoothed_wdontneed_reason_not_beneficial", "hrr", "msa"],
-        ["smoothed_dontneed_reason_not_high_risk", "hrr"], ["smoothed_wdontneed_reason_not_high_risk", "hrr", "msa"],
-        ["smoothed_dontneed_reason_not_serious", "hrr"], ["smoothed_wdontneed_reason_not_serious", "hrr", "msa"],
-        ["smoothed_dontneed_reason_other", "hrr"], ["smoothed_wdontneed_reason_other", "hrr", "msa"],
-        ["smoothed_dontneed_reason_precautions", "hrr"], ["smoothed_wdontneed_reason_precautions", "hrr", "msa"],
-        "smoothed_screening_tested_positive_14d", "smoothed_wscreening_tested_positive_14d",
-        "smoothed_travel_outside_state_7d", "smoothed_wtravel_outside_state_7d",
-        "smoothed_belief_vaccinated_mask_unnecessary", "smoothed_wbelief_vaccinated_mask_unnecessary",
-        "smoothed_belief_children_immune", "smoothed_wbelief_children_immune",
-        "smoothed_received_2_vaccine_doses", "smoothed_wreceived_2_vaccine_doses",
-        "smoothed_vaccine_barrier_eligible", "smoothed_wvaccine_barrier_eligible",
-        "smoothed_vaccine_barrier_no_appointments", "smoothed_wvaccine_barrier_no_appointments",
-        "smoothed_vaccine_barrier_appointment_time", "smoothed_wvaccine_barrier_appointment_time",
-        "smoothed_vaccine_barrier_technical_difficulties", "smoothed_wvaccine_barrier_technical_difficulties",
-        "smoothed_vaccine_barrier_document", "smoothed_wvaccine_barrier_document",
-        "smoothed_vaccine_barrier_technology_access", "smoothed_wvaccine_barrier_technology_access",
-        "smoothed_vaccine_barrier_travel", "smoothed_wvaccine_barrier_travel",
-        "smoothed_vaccine_barrier_language", "smoothed_wvaccine_barrier_language",
-        "smoothed_vaccine_barrier_childcare", "smoothed_wvaccine_barrier_childcare",
-        "smoothed_vaccine_barrier_time", "smoothed_wvaccine_barrier_time",
-        "smoothed_vaccine_barrier_type", "smoothed_wvaccine_barrier_type",
-        "smoothed_vaccine_barrier_none", "smoothed_wvaccine_barrier_none",
-        "smoothed_vaccine_barrier_appointment_location", "smoothed_wvaccine_barrier_appointment_location",
-        "smoothed_vaccine_barrier_other", "smoothed_wvaccine_barrier_other",
-        "smoothed_vaccine_barrier_eligible_has", "smoothed_wvaccine_barrier_eligible_has",
-        "smoothed_vaccine_barrier_no_appointments_has", "smoothed_wvaccine_barrier_no_appointments_has",
-        "smoothed_vaccine_barrier_appointment_time_has", "smoothed_wvaccine_barrier_appointment_time_has",
-        "smoothed_vaccine_barrier_technical_difficulties_has", "smoothed_wvaccine_barrier_technical_difficulties_has",
-        "smoothed_vaccine_barrier_document_has", "smoothed_wvaccine_barrier_document_has",
-        "smoothed_vaccine_barrier_technology_access_has", "smoothed_wvaccine_barrier_technology_access_has",
-        "smoothed_vaccine_barrier_travel_has", "smoothed_wvaccine_barrier_travel_has",
-        "smoothed_vaccine_barrier_language_has", "smoothed_wvaccine_barrier_language_has",
-        "smoothed_vaccine_barrier_childcare_has", "smoothed_wvaccine_barrier_childcare_has",
-        "smoothed_vaccine_barrier_time_has", "smoothed_wvaccine_barrier_time_has",
-        "smoothed_vaccine_barrier_type_has", "smoothed_wvaccine_barrier_type_has",
-        "smoothed_vaccine_barrier_none_has", "smoothed_wvaccine_barrier_none_has",
-        "smoothed_vaccine_barrier_appointment_location_has", "smoothed_wvaccine_barrier_appointment_location_has",
-        "smoothed_vaccine_barrier_other_has", "smoothed_wvaccine_barrier_other_has",
-        ["smoothed_vaccine_barrier_appointment_location_tried", "county", "state"], ["smoothed_wvaccine_barrier_appointment_location_tried", "county", "state"],
-        ["smoothed_vaccine_barrier_other_tried", "county", "state"], ["smoothed_wvaccine_barrier_other_tried", "county", "state"]
-      ]
-    },
     "quidel": {
       "max_age":6,
       "maintainers": ["U01AP8GSWG3","U01069KCRS7"],

ansible/vars.yaml

@@ -26,6 +26,11 @@ changehc_sftp_host: "{{ vault_changehc_sftp_host }}"
 changehc_sftp_port: "{{ vault_changehc_sftp_port }}"
 changehc_sftp_user: "{{ vault_changehc_sftp_user }}"
 changehc_sftp_password: "{{ vault_changehc_sftp_password }}"
+# claims_hosp
+claims_hosp_ftp_user: "{{ vault_claims_hosp_ftp_user }}"
+claims_hosp_ftp_password: "{{ vault_claims_hosp_ftp_password }}"
+claims_hosp_midas_user: "{{ vault_claims_hosp_midas_user }}"
+claims_hosp_midas_password: "{{ vault_claims_hosp_midas_password }}"
 # NCHS
 nchs_mortality_token: "{{ vault_nchs_mortality_token }}"
 # SirCAL