diff --git a/.bumpversion.cfg b/.bumpversion.cfg index af5b5125c..28df68b2a 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.16 +current_version = 0.3.17 commit = True message = chore: bump covidcast-indicators to {new_version} tag = False diff --git a/ansible/ansible-deploy-staging.yaml b/ansible/ansible-deploy-staging.yaml index 3056d79f2..3c04892ba 100644 --- a/ansible/ansible-deploy-staging.yaml +++ b/ansible/ansible-deploy-staging.yaml @@ -27,6 +27,10 @@ local_action: stat path="templates/{{ indicator }}-params-prod.json.j2" register: template + - name: Check to see if we have a secrets template to send. + local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2" + register: template_secrets + - name: Set production params file. copy: src: files/{{ indicator }}-params-prod.json @@ -42,3 +46,11 @@ owner: "{{ runtime_user }}" group: "{{ runtime_user }}" when: template.stat.exists + + - name: Set production secrets template. + template: + src: templates/{{ indicator }}-secrets-prod.py.j2 + dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py" + owner: "{{ runtime_user }}" + group: "{{ runtime_user }}" + when: template_secrets.stat.exists diff --git a/ansible/ansible-deploy.yaml b/ansible/ansible-deploy.yaml index f35aa40f8..a6dba64b9 100644 --- a/ansible/ansible-deploy.yaml +++ b/ansible/ansible-deploy.yaml @@ -27,6 +27,10 @@ local_action: stat path="templates/{{ indicator }}-params-prod.json.j2" register: template + - name: Check to see if we have a secrets template to send. + local_action: stat path="templates/{{ indicator }}-secrets-prod.py.j2" + register: template_secrets + - name: Set production params file. copy: src: files/{{ indicator }}-params-prod.json @@ -42,3 +46,11 @@ owner: "{{ runtime_user }}" group: "{{ runtime_user }}" when: template.stat.exists + + - name: Set production secrets template. + template: + src: templates/{{ indicator }}-secrets-prod.py.j2 + dest: "{{ indicators_runtime_dir }}/{{ indicator }}/secrets.py" + owner: "{{ runtime_user }}" + group: "{{ runtime_user }}" + when: template_secrets.stat.exists diff --git a/ansible/templates/claims_hosp-params-prod.py.j2 b/ansible/templates/claims_hosp-params-prod.py.j2 new file mode 100644 index 000000000..5b536ada1 --- /dev/null +++ b/ansible/templates/claims_hosp-params-prod.py.j2 @@ -0,0 +1,43 @@ +{ + "common": { + "export_dir": "./receiving", + "log_exceptions": false + }, + "indicator": { + "input_dir": "./retrieve_files", + "start_date": "2020-02-01", + "end_date": null, + "drop_date": null, + "n_backfill_days": 70, + "n_waiting_days": 3, + "write_se": false, + "obfuscated_prefix": "foo_obfuscated", + "parallel": false, + "geos": ["state", "msa", "hrr", "county"], + "weekday": [true, false], + "ftp_credentials": { + "host": "{{ claims_hosp_ftp_host }}", + "user": "{{ claims_hosp_ftp_user }}", + "pass": "{{ claims_hosp_ftp_password }}", + "port": 2222 + } + }, + "validation": { + "common": { + "data_source": "hospital-admissions", + "span_length": 14, + "min_expected_lag": {"all": "3"}, + "max_expected_lag": {"all": "4"}, + "dry_run": true, + "suppressed_errors": [] + }, + "static": { + "minimum_sample_size": 5, + "missing_se_allowed": true, + "missing_sample_size_allowed": true + }, + "dynamic": { + "ref_window_size": 7 + } + } +} diff --git a/ansible/templates/claims_hosp-secrets-prod.py.j2 b/ansible/templates/claims_hosp-secrets-prod.py.j2 new file mode 100755 index 000000000..b00b147d7 --- /dev/null +++ b/ansible/templates/claims_hosp-secrets-prod.py.j2 @@ -0,0 +1,11 @@ +class claims: + HOST = 'ftp.delphi.cmu.edu' + USER = {{ claims_hosp_ftp_user }} + PASS = {{ claims_hosp_ftp_password }} + PORT = 2222 + + +class covidcast: + HOST = "delphi.midas.cs.cmu.edu" + USER = {{ claims_hosp_midas_user }} + PASS = {{ claims_hosp_midas_password }} diff --git a/ansible/templates/sir_complainsalot-params-prod.json.j2 b/ansible/templates/sir_complainsalot-params-prod.json.j2 index 13f147bfd..d1d432446 100644 --- a/ansible/templates/sir_complainsalot-params-prod.json.j2 +++ b/ansible/templates/sir_complainsalot-params-prod.json.j2 @@ -43,100 +43,6 @@ "maintainers": ["U01AP8GSWG3","U01069KCRS7"], "retired-signals": ["completely_home_prop", "full_time_work_prop", "part_time_work_prop", "median_home_dwell_time", "completely_home_prop_7dav", "full_time_work_prop_7dav", "part_time_work_prop_7dav", "median_home_dwell_time_7dav"] }, - "fb-survey": { - "max_age": 3, - "maintainers": ["U01069KCRS7"], - "retired-signals": [ - "smoothed_anxious_5d", "smoothed_wanxious_5d", - "smoothed_depressed_5d", "smoothed_wdepressed_5d", - "smoothed_felt_isolated_5d", "smoothed_wfelt_isolated_5d", - "smoothed_large_event_1d", "smoothed_wlarge_event_1d", - "smoothed_restaurant_1d", "smoothed_wrestaurant_1d", - "smoothed_shop_1d", "smoothed_wshop_1d", - "smoothed_spent_time_1d", "smoothed_wspent_time_1d", - "smoothed_travel_outside_state_5d", "smoothed_wtravel_outside_state_5d", - "smoothed_work_outside_home_1d", "smoothed_wwork_outside_home_1d", - "smoothed_wearing_mask", "smoothed_wwearing_mask", - "smoothed_vaccine_likely_local_health", "smoothed_wvaccine_likely_local_health", - "smoothed_others_masked", "smoothed_wothers_masked", - "smoothed_wanted_test_14d", "smoothed_wwanted_test_14d", - "smoothed_covid_vaccinated_or_accept", "smoothed_wcovid_vaccinated_or_accept", - "smoothed_accept_covid_vaccine", "smoothed_waccept_covid_vaccine", - "smoothed_hesitancy_reason_allergic", "smoothed_whesitancy_reason_allergic", - "smoothed_hesitancy_reason_not_recommended", "smoothed_whesitancy_reason_not_recommended", - "smoothed_hesitancy_reason_distrust_vaccines", "smoothed_whesitancy_reason_distrust_vaccines", - "smoothed_hesitancy_reason_health_condition", "smoothed_whesitancy_reason_health_condition", - "smoothed_hesitancy_reason_pregnant", "smoothed_whesitancy_reason_pregnant", - "smoothed_vaccine_likely_friends", "smoothed_wvaccine_likely_friends", - "smoothed_vaccine_likely_who", "smoothed_wvaccine_likely_who", - "smoothed_vaccine_likely_govt_health", "smoothed_wvaccine_likely_govt_health", - "smoothed_vaccine_likely_politicians", "smoothed_wvaccine_likely_politicians", - "smoothed_vaccine_likely_doctors", "smoothed_wvaccine_likely_doctors", - "smoothed_felt_isolated_7d", "smoothed_wfelt_isolated_7d", - "smoothed_worried_become_ill", "smoothed_wworried_become_ill", - "smoothed_received_2_vaccine_doses", "smoothed_wreceived_2_vaccine_doses", - "smoothed_hesitancy_reason_dislike_vaccines", "smoothed_whesitancy_reason_dislike_vaccines", - "smoothed_inperson_school_fulltime", "smoothed_winperson_school_fulltime", - "smoothed_inperson_school_parttime", "smoothed_winperson_school_parttime", - "smoothed_vaccinate_children", "smoothed_wvaccinate_children", - ["smoothed_vaccine_barrier_appointment_time_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_appointment_time_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_childcare_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_childcare_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_document_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_document_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_eligible_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_eligible_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_language_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_language_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_no_appointments_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_no_appointments_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_none_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_none_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_technical_difficulties_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_technical_difficulties_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_technology_access_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_technology_access_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_time_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_time_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_travel_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_travel_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_type_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_type_tried", "msa", "county", "state"], - ["smoothed_try_vaccinate_1m", "hrr"], ["smoothed_wtry_vaccinate_1m", "hrr"], - ["smoothed_try_vaccinate_1m", "msa"], ["smoothed_wtry_vaccinate_1m", "msa"], - ["smoothed_dontneed_reason_dont_spend_time", "hrr"], ["smoothed_wdontneed_reason_dont_spend_time", "hrr", "msa"], - ["smoothed_dontneed_reason_had_covid", "hrr"], ["smoothed_wdontneed_reason_had_covid", "hrr", "msa"], - ["smoothed_dontneed_reason_not_beneficial", "hrr"], ["smoothed_wdontneed_reason_not_beneficial", "hrr", "msa"], - ["smoothed_dontneed_reason_not_high_risk", "hrr"], ["smoothed_wdontneed_reason_not_high_risk", "hrr", "msa"], - ["smoothed_dontneed_reason_not_serious", "hrr"], ["smoothed_wdontneed_reason_not_serious", "hrr", "msa"], - ["smoothed_dontneed_reason_other", "hrr"], ["smoothed_wdontneed_reason_other", "hrr", "msa"], - ["smoothed_dontneed_reason_precautions", "hrr"], ["smoothed_wdontneed_reason_precautions", "hrr", "msa"], - "smoothed_screening_tested_positive_14d", "smoothed_wscreening_tested_positive_14d", - "smoothed_travel_outside_state_7d", "smoothed_wtravel_outside_state_7d", - "smoothed_belief_vaccinated_mask_unnecessary", "smoothed_wbelief_vaccinated_mask_unnecessary", - "smoothed_belief_children_immune", "smoothed_wbelief_children_immune", - "smoothed_received_2_vaccine_doses", "smoothed_wreceived_2_vaccine_doses", - "smoothed_vaccine_barrier_eligible", "smoothed_wvaccine_barrier_eligible", - "smoothed_vaccine_barrier_no_appointments", "smoothed_wvaccine_barrier_no_appointments", - "smoothed_vaccine_barrier_appointment_time", "smoothed_wvaccine_barrier_appointment_time", - "smoothed_vaccine_barrier_technical_difficulties", "smoothed_wvaccine_barrier_technical_difficulties", - "smoothed_vaccine_barrier_document", "smoothed_wvaccine_barrier_document", - "smoothed_vaccine_barrier_technology_access", "smoothed_wvaccine_barrier_technology_access", - "smoothed_vaccine_barrier_travel", "smoothed_wvaccine_barrier_travel", - "smoothed_vaccine_barrier_language", "smoothed_wvaccine_barrier_language", - "smoothed_vaccine_barrier_childcare", "smoothed_wvaccine_barrier_childcare", - "smoothed_vaccine_barrier_time", "smoothed_wvaccine_barrier_time", - "smoothed_vaccine_barrier_type", "smoothed_wvaccine_barrier_type", - "smoothed_vaccine_barrier_none", "smoothed_wvaccine_barrier_none", - "smoothed_vaccine_barrier_appointment_location", "smoothed_wvaccine_barrier_appointment_location", - "smoothed_vaccine_barrier_other", "smoothed_wvaccine_barrier_other", - "smoothed_vaccine_barrier_eligible_has", "smoothed_wvaccine_barrier_eligible_has", - "smoothed_vaccine_barrier_no_appointments_has", "smoothed_wvaccine_barrier_no_appointments_has", - "smoothed_vaccine_barrier_appointment_time_has", "smoothed_wvaccine_barrier_appointment_time_has", - "smoothed_vaccine_barrier_technical_difficulties_has", "smoothed_wvaccine_barrier_technical_difficulties_has", - "smoothed_vaccine_barrier_document_has", "smoothed_wvaccine_barrier_document_has", - "smoothed_vaccine_barrier_technology_access_has", "smoothed_wvaccine_barrier_technology_access_has", - "smoothed_vaccine_barrier_travel_has", "smoothed_wvaccine_barrier_travel_has", - "smoothed_vaccine_barrier_language_has", "smoothed_wvaccine_barrier_language_has", - "smoothed_vaccine_barrier_childcare_has", "smoothed_wvaccine_barrier_childcare_has", - "smoothed_vaccine_barrier_time_has", "smoothed_wvaccine_barrier_time_has", - "smoothed_vaccine_barrier_type_has", "smoothed_wvaccine_barrier_type_has", - "smoothed_vaccine_barrier_none_has", "smoothed_wvaccine_barrier_none_has", - "smoothed_vaccine_barrier_appointment_location_has", "smoothed_wvaccine_barrier_appointment_location_has", - "smoothed_vaccine_barrier_other_has", "smoothed_wvaccine_barrier_other_has", - ["smoothed_vaccine_barrier_appointment_location_tried", "county", "state"], ["smoothed_wvaccine_barrier_appointment_location_tried", "county", "state"], - ["smoothed_vaccine_barrier_other_tried", "county", "state"], ["smoothed_wvaccine_barrier_other_tried", "county", "state"] - ] - }, "quidel": { "max_age":6, "maintainers": ["U01AP8GSWG3","U01069KCRS7"], diff --git a/ansible/vars.yaml b/ansible/vars.yaml index eaeff437d..c44844e8e 100644 --- a/ansible/vars.yaml +++ b/ansible/vars.yaml @@ -26,6 +26,11 @@ changehc_sftp_host: "{{ vault_changehc_sftp_host }}" changehc_sftp_port: "{{ vault_changehc_sftp_port }}" changehc_sftp_user: "{{ vault_changehc_sftp_user }}" changehc_sftp_password: "{{ vault_changehc_sftp_password }}" +# claims_hosp +claims_hosp_ftp_user: "{{ vault_claims_hosp_ftp_user }}" +claims_hosp_ftp_password: "{{ vault_claims_hosp_ftp_password }}" +claims_hosp_midas_user: "{{ vault_claims_hosp_midas_user }}" +claims_hosp_midas_password: "{{ vault_claims_hosp_midas_password }}" # NCHS nchs_mortality_token: "{{ vault_nchs_mortality_token }}" # SirCAL diff --git a/ansible/vault.yaml b/ansible/vault.yaml index 418d20a22..19b1388a4 100644 --- a/ansible/vault.yaml +++ b/ansible/vault.yaml @@ -1,228 +1,238 @@ $ANSIBLE_VAULT;1.1;AES256 -64323632663238643538393734386635383430343533663135396165656332643464383034353838 -3365616631666132306330363437303237333935393465640a396562623039656330396537626264 -62383239366136373130396337323635373937323636343637623663666232656161653536636333 -6637666136363430650a336235653663376439623862653962356166373930623632303830343937 -38323831656263303337643730356461303364646336653233346266363737386636633465373938 -66316363303766626562303033363237326662353164316364386430323961373938323263383935 -61383232303833353062363733636462356336326239626239393433343362303436633439613932 -63373430333736653935636162636162346137353431396635393537306632316537376539663364 -32333664303639623839323636653962363961626237663734343835303136636564653539366164 -32313265653339663335663831383030303830343338366465353131313535363536333032656336 -39346234303539326436393438633238376634373262323064643532373730303834653163393937 -32666663663763303035656237343866303533356635393031363763663235656637353162316162 -34333637363562386332303533623365323637396537616235326137366635666465656437353061 -37306536313333663163346435306563616330346639613838636430643761613066343862326536 -30633663333232653232623364353530326436656662383032356537366534333331633636306264 -63363834373464336261663639343636353530366633633034333735373532316130653639363232 -35663866616631376437323735396261303633623432623063376262333638303665363234333163 -32346534323132396162633565363238396336646137353937336330363531323031373032333731 -64373531303462343535386662643239666462653066393530383761643566663561663533613436 -37396535343039636231323865363663666137326262326135383131333035323765653236313062 -32333466336536636364383063326463373434336161376131663564366433636238323665373064 -36303831653138633931363365326337353634653431633832323833323730346630396534306665 -65333830636664323161363836666334353164653064646135643866383331326537623431613630 -63633539323263323963316566386633616133363330343332386137656631316162336361653130 -37303232643239616433633236636530343034376166616330336434616365386664666262666164 -32613562633566303533366335396464356361386237303264643733616535316132333832303236 -65666432633363363766366561313331393761636465313863383363656331353830656465643532 -31373335613233363462343664353033653731623231313039653064393163663136373937353134 -63376662376162653539303934313363353562333332663035633666633264636561636338636661 -32356461366235303962383163623464373231363261386437616132666637623466313236343766 -62363266663838346439656533643766393065383338333737643730653966626638373962333261 -37363364366161396139366461383739303266356334353466666431613061663164636363623332 -35313732313762663338353330326334613565313132396265636534363534623638326666376164 -33323637613634306134393161366439656364376436396130666130353739386335626136373533 -31323464343133656533616435626466633538306262353632323261326438623565353538633061 -31643937666432646631656366366132636165306239323337306337363333353530313764313738 -38373037623563633339323937356130306633383763646235343139353436393562663166333662 -32653432643032663634366563613263623862643563396332316430633637636664396232323064 -62363033633433353633366361623331643132656535306431323630646536316363373139313161 -36316537303537386266303762636435616632393635306439333338663238623365356462323962 -34333764346233313132323939666663656536326635656234613539313463313836626262303365 -64633863343333393566383239366637343061663632653236346563616431626230366635623965 -30353434623736666238333738383735663731343639313161313130336666356265386631393337 -64646637356230653463376538653730643435663139316130663738326331353762663435656233 -62653761306262666261303834326538393538303139663037626266623063336232653038393930 -61643235313062666638376532646630663461653531353165323866363839343162623366653964 -30313662343438613931323733373336616631623365393038633530633136613730653239653334 -33336263636339313163616335663530393038323430623331373962366638633961626439373631 -32343963366435666536343531313635393363613430623964613066363031623331313231396263 -66643766666335376332336335306536333965303137393836303061366163366364656461323139 -36376638626635623237646332343034623838623831653631663734313036303637326165326163 -31626138616636653835306261343631313031313731633262623237656466613362363963633632 -34656232656136373565376566343937356663386661316432353835663334633330323765393335 -65663632653535656631396266363163653637386233663463303237653364356261313232333730 -64386635326538666139333662336639353661386336613265613762626633376166363637623539 -36623265346337633936393561326261653961393932333933393964653565376535653835326339 -37316336323661316633623764363435313238313536336430373665636134393136623232613836 -35633065336532306466353136623765333662383937373763626331363536613430653630663835 -62336130316265306564363131313932613138343062343864336463333262316537376434353037 -35633536646336633564313536373637623062373361656631633731613265616562323130326163 -36383237626238626235386537396431326362336263333131386666373434393663643264343036 -37323862613130306332623130386635643533366165323034623966313634376564366637316332 -30373138303635323366366432663263353862363635623562666638346265653539366464626536 -33333330363138373361343966326464323461353836386339646533353234386132633862333963 -35356334343238336636323737393331303839326330653162356533393166353339346439366464 -65396434326339366562366563653861613130343963356639636462636634363230643134643030 -36346134336263303038643366336436313537636339643664353232633366316530376563376231 -61393436646163373238376264336330356530303564636566373930373135613135643962613031 -31666235623430626666316439316430653666343462326137373231396337643133373139396464 -63613736363938333766356662373162623231383833336364656534386435323633333465386131 -33323033373065363863373763376135376665313565363534656237643163323562646466626633 -33633239616335633031326638376237353639373733633233666136643135306332653163363966 -36626565326535666162313134653061333237366530366566653865393630306437303331356234 -33333661373462663434343933323861626334306539316635613366653432333564376333316234 -62326131633738636363396136383066316364353262363262356635323331346337396532323734 -31376335363663646436626361313164313363303262363364653966393936386465323935356435 -30633564396430353666656139396132343931316332633832336263383966303430356261366334 -39373339306337663132613737376235653863303061353464383035633761643864363566323730 -38346136316339613261303065303233633935303264346138383561383234636130343235303530 -64613631613731656338383833643138633632633532303131373962666265386665346534643032 -33633039353663613663616662636238363262646365363431386630363831623339303861636564 -63623538366433623833656362393764323166373632343939643761623538326561366137626365 -30656366373335396662663962343237383465303061356432636637313130396431623433363935 -37653830643564326662323964303831653639363131653763353161343663343465653131366164 -34356562343966326661623539646162623530366166366639373863393933366535383361303036 -64623166633135303038613262663533326238663462396637663634613436396338633362353134 -38633839396638366562396330633538363434633538633533316561616637316462323862353835 -36343761383231646265643162663635383638353362333634633738333438336463306132356639 -30343932616539376634316231633432646437326230613165316265353130633032356636376537 -37666661656665663230343363663132633839333431326634393236303731326634623063396439 -33633535666462623136366364613536343865313266396565303839396531643037616337653335 -34623864626562626638323039323466303034383962643733346466643366663564363039333164 -37303132346262383834303263643734393763346565373130336236393536633764366664323665 -30313232383636306163333830393262313937646137656538656565633539643066303761626537 -37356637643635666166616531303836363730333865616232383437616132383361313731666262 -33653638366534643630653137343838613834666461366339333663306233303239393666646662 -34623430333733643035366236616666333937663133323533376435613066656233386539656565 -31373261353764396433623264643238656130316666626339333837356639393739646134333365 -66653962663764643736373063306163383462343633396234636662643565353764636666303831 -39613132646130633162613233363739376534663838353366656262653835366466373336333934 -33396630396538316364316131316530323164333839363933373631373538313566333834353732 -62393132323839366365336565663936333831626665373631386533643862616161633438373131 -39663530616364633261666236343861366133393436326435373239333238303961353532323130 -33363037343161653166653634656531366238363337373863336634646539383262303335306537 -34666362396661356666373565383166633933323065363963383338633838643036633465663233 -39373537663262323735396331386631376436656335666635353136613232373461396131333839 -62643665326339333734626334363836366165373736386262333137313637333639336536633639 -30343231303934613237336434363836303761643436353566326233633865313936393033373831 -61306366323239323861623365643230623265336436366232653433623937393834313137363635 -39306531303861323065646564626261323435363536316337353862303033346664353533613635 -33336566396334623035373937306234356365626435353039366261313230383131313230303938 -30613435626561653036656135613561353838333833643035636335316661326537613034366635 -35343031336561623064366437363431636466646132333964336464326538306231643531666134 -33393135353130666466363131353764386631643235306266373265306434376362303761353633 -31323337313437316166386365313633323362363339656262623433663332366562653034343735 -65346430646635393334633262666631313432323930663563373734666335663862633932323162 -37396530373630376464356661633535343866336230396637613237636432303532326462393234 -37663433663663353338636361393136663133323131313231336234396363373063383333383639 -63366136346331326530653461383033656564363736626534383236643764653732633732323937 -64303263663163616636613665376361353630626133323330323438376461633065333439353533 -63303439303061323537636439383561653630353035303566366239336466323661623135313531 -62333036636532356165656134366332356436326134323430393566383765663439616431366330 -63363762363934316364343461313735333739633933613435646362353032636434383538353038 -64653663636162383035616334383239356437396265626462383737333636363566643231373733 -61613662666433363833633766636436643336386136366433313563616336383864323633353934 -61373335326138656566316336323563353037323134656336633237383665626266366131323739 -34613962336665363439646434333434626264353064363666353666636266633832633436633665 -61643439636662376465633163643965373339383435313933626231666235613137393130383962 -62316233643634336265663836386562643436663065373935623936366363313634616562366536 -38303134623664396361623732636562653130353538666432633263303731313063663135393535 -63623939646438633730353166353234653339373966333964323930386531663439386166663832 -61373835303533666335316137353162343933646565326662373935393864333961306138383665 -63376136336362376461303335666164666532393864323134303639663364343261366566333531 -38326333383830633734666366333731633261316435336136383438656566666265623737336232 -65623437383934333639373132613365633761363233326431666636393761343966343835383139 -34336330623735613163333164613466373965636264396666346534626265616264633738343666 -38643365333761646430636564386665656430383439356436393263396134636165373432396564 -33396261323964663736376432336461303064396438356566626230663961353363393961646435 -33323262643930343330333262616235313364643132356162313438633730626338326434386266 -30316261633737386334393138653935626665393733633362393662636132303937363538306134 -34336630663135373436373161343233656537643561346234323862313030653766666131326565 -34383238633966663132623763373338353533313161313166353861353266663866323330323538 -36626133623863366562343763363466393732346263393031323264356164373939346466653164 -65646436326265346465316532326534636465653265363765326262306136353137336131396564 -62376565386661646661666661376165623030363662633032346439376464643865656337626337 -31343335663933346339303666303838623465626334653864373766343265363463393861356561 -38613337383163633366396330636365306262323038393461303238396238363137383839633839 -33343534356339323933656433303937613930373535616437666138303064323733646261323131 -30663666636661653965366163336135353261376166336434616233396537343137363161393264 -63326336656235316466666134333233376138643331643839366337663830646430383863343537 -39643436323061326437616664313563313433643336366434353231306632616133383936393537 -38333665366333613361663436333963363132656633373165653737383334333638343062336566 -36656631366264363762613237356535316361376637663334326130386261626334613866613630 -39353739383565666237633064643235373061626361636261653033623231376435383839643436 -61336536643036663434653637386332656234323361336538643539353365333431326561623533 -35653262633737303663313735613131323733616565343264356334333562383738373133343865 -32383365306532316236386563303366323831386230623462633662643063646630393930633139 -37326263336464326433643764366433316431613733336662663034333131313638306632633765 -66633838633738353431663764613266623834353531613633643733363838363861626536363161 -35643233336464633336333236663466326538646230343464383263613839336630343838323637 -30376532636166323334656534353433316231306530666138636232393966653932393739373436 -64313935636462316534636466616335653535313037653261323364663666613633393337643962 -38633036306665326139373861353932633262333137333864336433393933376262313730326632 -61663339393431396137396362393937646164313965323430343062396434303238366435616264 -35636264613230653362336432633661626635353963633536386430653165363265653664633737 -39613833633366336164653062313130316533626333616132393264306237326538613337313531 -32366366376334653632393666646661326261653530663366626563613730313433303436303731 -36326536333739363833623931666330636166353238303333633861333662333363356566333437 -31636134333236353432386165656337306266303762363434366430383434653839306466366533 -62376630383861393536633166656330393439356564366566313261393338396563316265386434 -32363232373430343366373465323032356366626266656130623233386237306663393331353537 -35396538306632393934303663373635383964383230326334366163383932383835613235633764 -37646466633037336131623336383238363239366534346631326336376230313861326237373930 -37346662653064373734653335643164396530333732323064636264333563363532313630366361 -35663835396261633337626130356430653038313233656262313661643636383037633135633862 -66343261383964343461353535616164303934643863336465316432383364363733326634376236 -64336161646565653335646130336239303739353230313264346536393262326134333966316666 -38636536626134323739306139376630326432626537343738636131663832636337373533316233 -63383434356330336430393331343062333731653635316232636264616663333366376138373865 -37653433643830323431383265333637363337623866626134336437623062613236346230313362 -35316330316334383633643136386664396532376330636337376630356634306532383662633234 -36346163393335356532653038313036653331363039653034363333333138356330326666323135 -63653263393762303566343931376165333237393234613734363662343132396261306363623764 -35613238653834323739643738303665376366653332623134616533376138386435633966343261 -62663139393736646633386536323461633263326263626361363634653137616539336633626333 -65643538623639333732306436626164333439386565653435356363626636623233613761626435 -65373039383134343238333161663731313738363266346330306235323334633230633235363962 -64626337643463303863393566343533333463373036303339336365303765623039333439303066 -34316339616531376139363239313932353863373062333930613630323163333532363538373334 -62656635643930633333666163376662623662363166636430663562643262343361663438656336 -64663965333933363565313836303531363261383733346139356238316464393933353763613361 -35323265353936623263393231343737373166363730393338646137646632373838333363303865 -31643633366637396663393964326562366639343331663664666261646638663631303631346332 -31666330383639303432363361326365363361356565323766616537633063643933666234343265 -37336131353636336334653635663638383065623838386161626564626534343932366230376563 -61616332656666313439613663333034616136323134386535633638633833636235383666653064 -35343732353237373831353263653637383263356235306231393030356230653934396261363661 -37623932616139373466656233303763386462303462313063393331613732373265336436643730 -32313634666564646161616330653435356264623333636132646665623066393135356231636333 -64616432333139653563323136666364373232663766383930613234383637333562333635666134 -63393665323833373439313538363964363230653264303263626661393837656339636235363164 -31343965633537336163313432306132363861623532396661666630643263343338393463626565 -32323564306466396166343431616236646136376664363436353464333232653431383135356164 -61643461383534366365343231646439396637623262356539313463303137313033363237363464 -35366432616339366538346235613561323862333230343863626530663065333732333730323232 -64626162656162326566326665323435626435353766626166323633353337313334313761356230 -66663434333765663164373563363136396538613265343865623831663939643036656330633065 -39363966623932323639346239336334383932663766613134353166626462353466373333613435 -36366166346339303730333435656464356236383461356166613134396339383763336636313165 -31613338366331333036386433343564393031663431643364396633323036656464336139363265 -31663239366265323430383462303334316238363834353431386261353930323437346461366334 -39613761666332346331656136653033353463353732346430383665666665663938626264373135 -38623332383734346264333465386463356662616236646463333765303361353536646239343362 -33326438623865646561333632646233363461383734646332363534393730343535396338303435 -33373362313564306633393030326633376438353035373566656534333236643766646630303535 -37336464636563376439323166313836656365646461383830303864386337643561616264393737 -30393332626465623934353930613837656639386235383465343961333732656239386631336337 -62393262363164643430633434333662383764363665393939316365626137376166303838363065 -37383731333965323337383931633566356237336464666436343031643864373333646130346364 -39633661313238626565323135653134386137393036393930613766663437646263343761303237 -39343638633366643937643866613731623230643862313336366161356432653535386531303430 -63303762666465626638393335333863396665626333303038373332336131623461366361396565 -65356431663531366436626464343230613931626163393235626332353234323231646464353037 -35386566326363366436333135343139643666616233326234313838646436333861653666303132 -65333839633434623961393366376538653931356262653534303232633137653339346262313166 -32396439303339343761623939636531343630616461376462313335653366623063643231666162 -633664356630373563346630386236376239 +62396535363664646663333461353838663562396536653339336637626532623035626264333462 +6530386466613032306364343565613932386230333461300a386438383562396162353031306262 +36313763643039653632363966626231386532616636653464393338343332376530623236623261 +3734313064626130310a333433343239356665336630613437373765363137663531376430343834 +66636663373961363239653636323364633739333266356639386663346362613866363432353130 +65373665326332656263666338313433333564663166363630356336343932633261633439393030 +61626436386666313137646335306266646564663766393936626461383534356631656266313639 +37373733646338356562376230343734376637646562646161376563313262363666363164353362 +34363834376466633962663266643736323439323263383862306663336136613263626632623363 +63616363313338646634336133306230333839323437326339616530613566393363616232303636 +31383634343333306661346333383133613430323038653139346565366365343438336362613962 +63316164396464333538666336313138313234366261633232336630316466353733373364653036 +32306465383066396165633732653731373230376263353966666433396562316565343732356565 +62326539323039663837636464313565303531656238323234393533353466643935633239313966 +64613866393963316434303334393636366161633134343035396636313437366636323732306639 +34396463396438323061646635626230343239366232376639386566306666643035346336323865 +66333739393731633166633632306331303033653735346430363662383064383534373236313338 +34343337656638373365666232633732323733303337656234616531343964303737613562643730 +38633237633165393738623363393863656333666431613637396362343462666133336435306366 +61633865356336616433626666386264626636653436373331313461313061613234316165363264 +39633162333133373365306662303464353634326163393834373461356565616463376639336361 +64363865396230623638343139663963616531383639373233323131656662366130376433303935 +61396132643762396435373762346635663833363634616336366266623938386134346633646533 +35626164663234306164313935393430646464326135353338633931336565623538313334346366 +65376633346436393735373535613730633338663838303731366663386638386234336465653363 +66653963333332633132663030653931666438306436363666396234646138363639643131303733 +37313032333931616363663362323866613861613932313737663162613765326337643136396130 +36613361663836653239356137393036376538613035303765623932613965376337383636326262 +64343237386561303164653765313436333436373431353630643666353763383933623635636333 +32343039373535643161356431613266623834663635346137386330326637653931376336393564 +30616432623736333235373139623633366534336462653834656630313832616561663630656336 +66633634313465363537633866633830363930613634646364666231626333383461633839356437 +34643634333531633766326537373734613465316335333064313762313037653064353331303066 +30616566336565353831306330336533366465326366636132366636333032653165326233326336 +64613938356238646637666464366562303234306261643130616434386263623264646365653861 +39306332623832656237613062386264393464656165623034373866396437323036376439386639 +32343265623035666638663665333434346564346439643965306163333434333062366261303636 +61313330363436646236343039306233383161623434616662646262306133656231633037393364 +36353862306434633362393564396638323864383439383134626562306163633038373736303137 +38383239376431393830373963363763313935343133313333346662326434313239366632613030 +65373165656438653734666237353365373937656562376163313961306465663332316639316162 +30653734653762346138313164353461373139366130363337363633633736373639363934653539 +64623830613035636466663462306238646162663931336436653838633338646161633737336162 +64393531663564613965333363623065656536303461656536393963623139353662386461613565 +65303635356333333535376662616232613534313563613463363732633361376436613464393861 +65383630343431623136353938346566383036396238356232643064376564626139343463633638 +64613865343839386630323934326631643533646237336633333534306661336662616132396534 +30326232383832373063643065373163326163383836646164656661396432633039613561663531 +32343062613062373135393366633136336433323534656539393232316662656134623431373832 +63303261646236313839353237626465323666653839383031393164343636303765316261373136 +38643664333338326366613938313732376535363666633034323337323863633664386563616662 +33356361663530353937373132343166623035646139393662663035313162343538333837313861 +34393630653761383632636331373065366338343538306339303434363139646137636364666131 +65616135323732666430366539653164343638343365633237356564326434323439363735383133 +38336535386432633661343039303861343562393962346131666438613463333237393837386233 +34333932363333313362366432616461663463306638623134333737366433663164376663323036 +61336636616236383736393734373231656435303532616237366562396265656462363432623237 +63313638323239613134646435623061626630366533623439373535666666363463303261656538 +64336237633830613163313161373931363634653331663865383664383561663933666432393036 +37303164303435323734383962353963323563386463346666346631366162373334356164363033 +34313336333863383932616239306635653563333531323465363361313564663933326130303661 +31373763326535336662356337646261346364323732346137303939393931646663366161373732 +31376534333631386539363137356463623332363234303933366639663431353236376334363135 +33323135383138393039313033333365613563323161393466303233343537353530386636633262 +36656139613639643161616630643162643032613861316434343665396166343062643431613130 +66303237663666366361633861383938343466393437326133316235653434656233656435636137 +35643561626539653232353832333934326161653664666664653163306634626533343034326266 +36323332363936623165616337306135633638653836336332643766313837353539313062663234 +65333039383833623464333039343662623138636334643361646265386265323134306137353830 +36303830363635623065336235663436626234366637346463333762623639336364613437313134 +38396236616534303861353831663366626539393637306563623864616532646530313634663339 +38326137646365306461313438616337373431396639636131303835353366333139623038636337 +39323666633161666638376564643537623765356563643133316536646565376336316665626630 +33303136633137326232626638633830393564303036656534373432376330343663316664316531 +37386332663134623735333538626630396464393030366539633336316563323532613136333032 +31666236643530363363323761373838653662313632613664656264633831396332373166613030 +63373566356363373934653831393239613534373431346530646533393235323764343131376236 +62313231643238306631633763613139383237623338386239336232653166663938306636363431 +63626339343738623733356130303936633265353061336136633562653665363835613135393865 +33373264356437653730333163373032376364656538653331613461663063623535396238353361 +30396331663836323564656139353930336436616161633138376532633339313033386333383165 +34663138333866616133663030323032326334653130383938643638376439333037343032663930 +61313233663961633138613639666130633035616163373332653139316161393765396530616138 +32356436366265366431636663643066633634643938303335656331326363633164333934336165 +39613235306336323866343364356666306663663632386338626633313962643663363364643539 +34306338663335643337653732396139613938326237366164336432326538326665623537323730 +33303930326438656636636634626532323562333531643435383363376535646665363335383832 +38383565346338393865303930333364643832633532373362343236303465663564663834376439 +33383566366463343330653536643964643466303562343733653261636538393361626131333964 +33653935373930333535373237393962353262666265646462366561383434623231636538333163 +61393335663161386566323463616331346636373931333632316162343131633631643837383437 +32313037633463633666326230313731343231343061616566373034656663616462343031373761 +31326533653134373936363435643938363531343963393461633132313864313162393764366464 +35393739326266366632633535323965643132613537646161353236393961373436366432306638 +61326365633964656531363537663334333638303239653732393833353931666263343763366636 +33643031333534323239633730636231346531653936363337333731333035336233343830373639 +35313763643133373362383061303332306639326264656364353562396335373237346664646330 +33306163623563626337366135346333343339656233343235313362623836303361386664326161 +61343639303633376333373337353730643731343066393332343365626330383935646234383766 +31396264306631363538353461313436633561393738636634366332353037633063333933316661 +64643163383435323763366635323130643337653961363063613038393361336335633366623933 +36663264393536623962666339316363363164383937623062653732623664353439343234386166 +63383563336366356236396235393862633865326439396235663639646266626264623465623338 +36303466303862356438633764623362383830666233323963356161343264393132333064393639 +30333435633833643566323762353965393764376365366135326430393238633632333236376564 +37366361366338386264633336643265306239663836313334643265303536666164393334383538 +38353861643334636634326363646435353332346234313331356536666132326161343933393761 +31653461336564616561396438313163643239373961343231303132303339353934336432653063 +37633530666663633338653536323830346232363135666436313362386461313865373362636262 +65363461386663366462646565666566333333633664313837333533646262323865666136623339 +39313337323736663731353231313064633936626263626665313635386435303965363262346461 +39313464356635333666333766653232373866323131336139306235623133613964646663396534 +65643765316363396235666632626238643966633765353438653164626266323766623863336335 +62376464336333636262386230393165363933323461636236373439383965303564663037396139 +65643439346331646239313561613262343563656138646233363234313239643035663936633231 +31643465656561386336373364643764303063666135663962313439343936393661613762626432 +30643133346161666364363162616462366330303562666565613665383364343837306661353332 +64336665373036383331376366303431313039663161373538383065323438326465323039383264 +63303636636332623864393937643464343933306361636661313235363034323666303438346232 +66336661386265383263313236356562373032633530636335653231333534623035303862613264 +32303630656565633538346430373731386433633363333636646235386336633834613839323362 +32386634316262646362626264383730356364333963363038393430373733333638616233303862 +65333765613362653935343361336533613662363533643039346663313733623633343739376438 +37383034363430623934633436656634366634353266323462336265623763333463333635306162 +31313230333433336561316432623334643663333233616231653937343239386563303930623635 +37663066336136653566346131623336336435613431326466626634326532356235323837656235 +36383464643938643833363433323263346536326536633863393833376162666566396137373637 +39326331656630343065393261366462646662643633616231663634373439343135616361666634 +61643837656532653533383261313632313637663933646138356333666335666335666364326265 +36323262386166323833376463396635303231306131366563383864333436366165386630386165 +62306634303630373565636333386262626138613165376632633665636265323864663132616538 +63386463386663616134313531373961313331323362653134396461343333316265306134383233 +34626231663237643861646636666635656432653531326630613334643935316135666164643237 +33313166373732623463313637383933613632346133356332633739653337373738346134643966 +33346638336463623736393961396333626362373133633836643666653130626334666233653362 +38316235656334383363663463346465613439393762323264613066303037353634616566326635 +65316334613738616261636664393333303034326561653637333264663965663365663939333636 +66643063346461356636363862346339653564333463666361353931393432313438353766656235 +64356331396237316366346331663232303139386536383434323161303362623762396666356166 +38353165373535323437613464343061646531663330373130653132663837623465646537653230 +37383666653332336362366364623935303166386638613261613430626138653233353332336537 +30656634396333363339306638316432663165343631316365613265613166373662383732376635 +36613831383338623539613264636234323336633839666135343266313838356333363031316264 +61653935343034613733616131666531303665336531323632393033363337386162623464306163 +36666531623437363066323033373065626538326564346435613230363765613335623237333337 +33646530356537646362346562356433336564336563393938303234356266626433306138396636 +37363765616135626236313531646530623235333630313330373434643239323633306130393462 +38386438393536303835363633633235616266663034623430346462663866303163663035323933 +38376363373366666564303335613937626166343666393963353336646530393465316539656663 +39333432653566623062373939353166323666663938666364633636333531326637306539653365 +65633066373965663964666463346565303166346664383462383737336466636231323037626332 +61313034353530623663306361663639613162323735383962393364646364333430383263306639 +65373037633931646139303732653964396436663536343938313438316165303231353134623136 +36333265323738383236393266393738633762626138386162643538393163663765653737313062 +63386632623962366236356161366262353336366563636438353333363731333264363666346564 +39633063613635366139316630323435623031353661373666666665636437383566393430666461 +39356437306430656536396339373163663162613061653166333434323261626566313239313631 +35616666623637313166306539363462316436646664326439363538316365393462336134393463 +37333431343639343930353564633031336562316233316131323931626466393865313231376666 +62303936323865383131633539333965366238373461373337636532616431643464393338646437 +65613830303838393664383434636433343132653765356434646633643634343965343138313333 +38326131396165643035336361353365633466643461343464643235653336393361313765636133 +39393037353831333666356532336330333061626361373064386163633766363937656365636638 +33393764653338336234343237643634343639373738376162303961653039353262383133306364 +61646664316333613730616338346363346332636661613661303962363236336437393634326330 +34306561336264396261633932373734373862613732633462616463346538313231653362653561 +32356663373862633133373364373139633865356366363064376432353331383234623835323662 +39313535306535396462653631356565303637343561616365396233343731303938306165666264 +33623136366133633761333833663835656132376331623939616235633336616166633537663834 +39346437646338646564303036383432323437356462373830633234386265666166633933653339 +30373937393962333365366239663261643237383865323836623739656335303237316538653364 +34343339353732653232636338616461313034393432633564613262396130333531633439653939 +65636231376261613638383035666438346539656236653831396431616563323463616330323062 +33313562623739323033663038613838633765383330306632613331326239363363396335346365 +65383033333262346565343262353062373865393134653639333934636537323433373766333535 +34373864306661336336396139336162666537316663303566343934666138333635643137646336 +36333339376362643165363337626430636130336363323835396166616338656635666533333763 +61333834623437613166326433393931336633656362313265383735653536316663643636366565 +38633330363036326465306136363663363363316230313936356237333266353266383863303835 +31363137643965343462366166323363613636633033373139616262356435376262323831386436 +63643734333561653465373265636663663033333638623438356365326633663966633039316338 +62653962666230366435313362363566333339336532653561333265323731313030616261306263 +37306131643164366565363431306161323535323536626162653765633364363764663866326336 +37316333323862373433326536623166313763353561383138333336666235333538623866363239 +37323234646438336661626362663862313937353430366566633139356138623839393938353838 +37353939363564656261633561326162346464336664303230336464646630363665306230316666 +36386264623631663663346438613333346635343735386536353433396534353862363165313034 +65346464633665313163333339353138633332393634306239353261633161633062623130316461 +32353538356239346262353630353130656161313336343364396433646461643064363233303764 +32353734663465356165346634626363373665373634643561326639643432346439646463346335 +39373836326664323639333064616564363161376434616237323366623836353134396637333039 +34656130373036386664363834636333363065623730653664623364316535316462636431326466 +32653332373766303439366364393831376439356530393031643262343534353530633066626533 +34393730663639306165656630333631306337656334616334613765333364613538613236376262 +38623366393239303161643639373937313661636537393931616135356536626238356462396539 +65316137373761313935323337396436343130316366316337653664633339623464393331336138 +39643933626165323331303732633335303965373262353962393962313761326163306337306134 +38343164343937333431626165313166666461363431613136323539653961356137343437666136 +32333830316263376337623564323035653934386162343263663265383765366361623030373433 +33323265313230623439323365623037386135376430663133343037663964326464633965323766 +32626430323863623930643730666437366538366363356332613932636331376533313439323935 +65303563623037396434376533303735393262373561313535636334383833663135656663643639 +63386231323866303935326262623866343661393230666135653761333731353763623564656234 +30623039366535376266353837653061363330663938363362653237303437393862336665623632 +66396462643431366438373931646166323665653537653762363436376565323239313363303866 +34313462666363633230303861313666383031303838633961303537633037323933323632626438 +35386464646235656334323065306134646634373538326435356134393361663038333037643463 +36326530633735643839323634373061343962366234333634616633633338326437393737653866 +35623631383434646435613666646334306433636139386236373766366464323739343337333765 +61373566303035376434343638326562316463623532323531616436356137623339376132363664 +34376363326664663263396464376232666263396264336130663530383539323836313465643863 +32393065656134396539303232303937623663353239653230346334366635623231363264323362 +37323139396639326565356563336665633131303361313331653938383437396434343231613664 +34353931613337636133323062383335643339323466343735383138663164633866326433366337 +33626431343233656637663533656165646338336333313266653339653330343661373735306236 +36663735623131383830636439383763383166643332316130306262306462373435306363313832 +33663836653462626632656337616166613165633133343163353139323032636237356665323836 +63663462363230613433343934643435373233353239303931663031653833373638386162343766 +32616430373433663831343837636464343262306337396262373163316236323839626137326166 +64646536356466613238646434633737343661643966356539363361326661643066653466353237 +64633664323030323135383663393265383666376164393237386535653161316663666437333538 +39356562353237396438633739393365613135333032643662343462656132613538383664636434 +39363563643436393461656535313461626235303762613532646164393932613333333234336430 +38643862653563343434326365643330646665396662313865396136376130396465383630323330 +35346331376433663264663266633230643731393463356262333764613461303930376330643564 +63356666353536626439653535616634633462303236666239663263363438663162666663393935 +62363930656465383836333830396561386138386331653063326134386238386332643730623739 +66373464343739323366313734336165623237373664623564653134363034653736306132373734 +35373532323763333337393337306264636436376462326331643163656635323662366337306361 +64336362366131303632616232323937386239663433626632346335666264353531313361653436 +66306664343530613833663433333932363332386338646137636362343636623361643063626564 +63613833346161666131646231333161313162303564313461323137313439353862363034306262 +37373336366333663131316466663962346536656265353735363764306131613333336234643335 +64346633323737376166616163643863626362306566616164336232303637623130613237303934 +36396432366662383835343632343062616230383038653362316231343036623139363834666131 +30393731323434343061623132363336383039366432346337383031366135623335336263353532 +38383466613930386162306233653438303030623966323061636537393365373632 diff --git a/claims_hosp/.pylintrc b/claims_hosp/.pylintrc index f30837c7e..7fc2f5c30 100644 --- a/claims_hosp/.pylintrc +++ b/claims_hosp/.pylintrc @@ -7,7 +7,8 @@ disable=logging-format-interpolation, # Allow pytest functions to be part of a class. no-self-use, # Allow pytest classes to have one test. - too-few-public-methods + too-few-public-methods, + [BASIC] diff --git a/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py new file mode 100644 index 000000000..6c2a3f184 --- /dev/null +++ b/claims_hosp/delphi_claims_hosp/download_claims_ftp_files.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Downloads files modified in the last 24 hours from the delphi ftp server.""" + +# standard +import datetime +import functools +from os import path +import re + +# third party +import paramiko + + +class AllowAnythingPolicy(paramiko.MissingHostKeyPolicy): + """Class for missing host key policy.""" + + def missing_host_key(self, client, hostname, key): + """Check missing host key.""" + return + + +def print_callback(filename, logger, bytes_so_far, bytes_total): + """Print the callback information.""" + rough_percent_transferred = int(100 * (bytes_so_far / bytes_total)) + if (rough_percent_transferred % 25) == 0: + logger.info("Transfer in progress", filename=filename, percent=rough_percent_transferred) + +FILENAME_TIMESTAMP = re.compile(r".*EDI_AGG_INPATIENT_(?P[0-9]*)_(?P[0-9]*)[^0-9]*") +def get_timestamp(name): + """Get the reference date in datetime format.""" + m = FILENAME_TIMESTAMP.match(name) + if not m: + return datetime.datetime(1900, 1, 1) + return datetime.datetime.strptime(''.join(m.groups()), "%Y%m%d%H%M") + +def change_date_format(name): + """Flip date from YYYYMMDD to MMDDYYYY.""" + split_name = name.split("_") + date = split_name[3] + flip_date = date[6:] + date[4:6] + date[:4] + split_name[3] = flip_date + name = '_'.join(split_name) + return name + + +def download(ftp_credentials, out_path, logger): + """Pull the latest raw files.""" + current_time = datetime.datetime.now() + seconds_in_day = 24 * 60 * 60 + logger.info("starting download", time=current_time) + + # open client + client = paramiko.SSHClient() + client.set_missing_host_key_policy(AllowAnythingPolicy()) + + client.connect(ftp_credentials["host"], + username=ftp_credentials["user"], + password=ftp_credentials["pass"], + port=ftp_credentials["port"]) + sftp = client.open_sftp() + sftp.chdir('/hosp/receiving') + + + # go through files in recieving dir + files_to_download = [] + for fileattr in sftp.listdir_attr(): + file_time = get_timestamp(fileattr.filename) + time_diff_to_current_time = current_time - file_time + if 0 < time_diff_to_current_time.total_seconds() <= seconds_in_day: + files_to_download.append(fileattr.filename) + logger.info("File to download", filename=fileattr.filename) + + # make sure we don't download more than the 1 chunk (2x a day) drops for IP - 01/07/21, + # *2 for multiple day drops + assert len(files_to_download) <= 2 * (2), \ + f"more files dropped ({len(files_to_download)}) than expected (4)" + + filepaths_to_download = {} + for file in files_to_download: + flipped_file = change_date_format(file) + if "INPATIENT" in file: + full_path = path.join(out_path, flipped_file) + if path.exists(full_path): + logger.info("Skip the existing file", filename=flipped_file) + else: + filepaths_to_download[file] = full_path + + # download! + for infile, outfile in filepaths_to_download.items(): + callback_for_filename = functools.partial(print_callback, infile, logger) + sftp.get(infile, outfile, callback=callback_for_filename) + + client.close() diff --git a/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py new file mode 100644 index 000000000..e417183c7 --- /dev/null +++ b/claims_hosp/delphi_claims_hosp/get_latest_claims_name.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Return the latest drop.""" + +# standard +import datetime +from pathlib import Path + +def get_latest_filename(dir_path, logger): + """Get the latest filename from the list of downloaded raw files.""" + current_date = datetime.datetime.now() + files = list(Path(dir_path).glob("*")) + + latest_timestamp = datetime.datetime(1900, 1, 1) + latest_filename = None + for file in files: + split_name = file.name.split("_") + if len(split_name) == 5: + ddmmyyyy = split_name[3] + hhmm = ''.join(filter(str.isdigit, split_name[4])) + timestamp = datetime.datetime.strptime(''.join([ddmmyyyy, hhmm]), + "%d%m%Y%H%M") + if timestamp > latest_timestamp: + if timestamp <= current_date: + latest_timestamp = timestamp + latest_filename = file + + assert current_date.date() == latest_timestamp.date(), "no drop for today" + + logger.info("Latest claims file", filename=latest_filename) + + # return for other uses + return latest_filename diff --git a/claims_hosp/delphi_claims_hosp/modify_claims_drops.py b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py new file mode 100644 index 000000000..0ab93ebcc --- /dev/null +++ b/claims_hosp/delphi_claims_hosp/modify_claims_drops.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +"""Modify the drops. + +Drops are expected to be numbered as: + +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_1_07052020_1456.csv.gz +../EDI_AGG_INPATIENT/EDI_AGG_INPATIENT_2_07052020_1456.csv.gz +... etc. +""" + +# standard +from pathlib import Path + +# third party +import numpy as np +import pandas as pd + + +def modify_and_write(data_path, logger, test_mode=False): + """ + Modify drops given a folder path. + + Will rename necessary columns in the input files, and check the number of + columns and duplications. + + Args: + data_path: path to the folder with duplicated drops. + test_mode: Don't overwrite the drops if test_mode==True + + """ + files = np.array(list(Path(data_path).glob("*.csv.gz"))) + dfs_list = [] + for f in files: + filename = str(f) + out_path = f.parents[0] / f.name + dfs = pd.read_csv(f, dtype={"PatCountyFIPS": str, + "patCountyFIPS": str}) + if "servicedate" in dfs.columns: + dfs.rename(columns={"servicedate": "ServiceDate"}, inplace=True) + if "patCountyFIPS" in dfs.columns: + dfs.rename(columns={"patCountyFIPS": "PatCountyFIPS"}, inplace=True) + if "patHRRname" in dfs.columns: + dfs.rename(columns={"patHRRname": "Pat HRR Name"}, inplace=True) + if "patAgeGroup" in dfs.columns: + dfs.rename(columns={"patAgeGroup": "PatAgeGroup"}, inplace=True) + if "patHRRid" in dfs.columns: + dfs.rename(columns={"patHRRid": "Pat HRR ID"}, inplace=True) + + assert np.sum( + dfs.duplicated(subset=["ServiceDate", "PatCountyFIPS", + "Pat HRR Name", "PatAgeGroup"])) == 0, \ + f'Duplication across drops in {filename}!' + assert dfs.shape[1] == 10, f'Wrong number of columns in {filename}' + + if test_mode: + dfs_list.append(dfs) + else: + dfs.to_csv(out_path, index=False) + logger.info(f"Wrote {out_path}") + return files, dfs_list diff --git a/claims_hosp/delphi_claims_hosp/run.py b/claims_hosp/delphi_claims_hosp/run.py index 58cba1c56..6c7405a36 100644 --- a/claims_hosp/delphi_claims_hosp/run.py +++ b/claims_hosp/delphi_claims_hosp/run.py @@ -7,6 +7,7 @@ # standard packages import time +import os from datetime import datetime, timedelta from pathlib import Path @@ -15,6 +16,9 @@ # first party from .config import Config +from .download_claims_ftp_files import download +from .modify_claims_drops import modify_and_write +from .get_latest_claims_name import get_latest_filename from .update_indicator import ClaimsHospIndicatorUpdater @@ -31,7 +35,7 @@ def run_module(params): - "log_exceptions" (optional): bool, whether to log exceptions to file. - "log_filename" (optional): str, name of file to write logs - "indicator": - - "input_file": str, optional filenames to download. If null, + - "input_dir": str, directory to downloaded raw files. If null, defaults are set in retrieve_files(). - "start_date": str, YYYY-MM-DD format, first day to generate data for. - "end_date": str or null, YYYY-MM-DD format, last day to generate data for. @@ -53,11 +57,21 @@ def run_module(params): __name__, filename=params["common"].get("log_filename"), log_exceptions=params["common"].get("log_exceptions", True)) + # pull latest data + download(params["indicator"]["ftp_credentials"], + params["indicator"]["input_dir"], logger) + + # aggregate data + modify_and_write(params["indicator"]["input_dir"], logger) + + # find the latest files (these have timestamps) + claims_file = get_latest_filename(params["indicator"]["input_dir"], logger) + # handle range of estimates to produce # filename expected to have format: EDI_AGG_INPATIENT_DDMMYYYY_HHMM{timezone}.csv.gz if params["indicator"]["drop_date"] is None: dropdate_dt = datetime.strptime( - Path(params["indicator"]["input_file"]).name.split("_")[3], "%d%m%Y") + Path(claims_file).name.split("_")[3], "%d%m%Y") else: dropdate_dt = datetime.strptime(params["indicator"]["drop_date"], "%Y-%m-%d") @@ -88,6 +102,8 @@ def run_module(params): weekday = params["indicator"]["weekday"], write_se = params["indicator"]["write_se"]) + max_dates = [] + n_csv_export = [] # generate indicator csvs for geo in params["indicator"]["geos"]: for weekday in params["indicator"]["weekday"]: @@ -114,12 +130,27 @@ def run_module(params): signal_name ) updater.update_indicator( - params["indicator"]["input_file"], + claims_file, params["common"]["export_dir"], logger, ) + max_dates.append(updater.output_dates[-1]) + n_csv_export.append(len(updater.output_dates)) logger.info("finished updating", geo = geo) + # Remove all the raw files + for fn in os.listdir(params["indicator"]["input_dir"]): + if ".csv.gz" in fn: + os.remove(f'{params["indicator"]["input_dir"]}/{fn}') + logger.info('Remove all the raw files.') + elapsed_time_in_seconds = round(time.time() - start_time, 2) + min_max_date = min(max_dates) + max_lag_in_days = (datetime.now() - min_max_date).days + csv_export_count = sum(n_csv_export) + formatted_min_max_date = min_max_date.strftime("%Y-%m-%d") logger.info("Completed indicator run", - elapsed_time_in_seconds = elapsed_time_in_seconds) + elapsed_time_in_seconds = elapsed_time_in_seconds, + csv_export_count = csv_export_count, + max_lag_in_days = max_lag_in_days, + oldest_final_export_date = formatted_min_max_date) diff --git a/claims_hosp/params.json.template b/claims_hosp/params.json.template index d6df27ed3..e200fa8fc 100644 --- a/claims_hosp/params.json.template +++ b/claims_hosp/params.json.template @@ -4,7 +4,7 @@ "log_exceptions": false }, "indicator": { - "input_file": "./tests/test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz", + "input_dir": "./retrieve_files", "start_date": "2020-02-01", "end_date": null, "drop_date": null, @@ -14,7 +14,13 @@ "obfuscated_prefix": "foo_obfuscated", "parallel": false, "geos": ["state", "msa", "hrr", "county"], - "weekday": [true, false] + "weekday": [true, false], + "ftp_credentials": { + "host": "", + "user": "", + "pass": "", + "port": 2222 + } }, "validation": { "common": { diff --git a/claims_hosp/retrieve_files/.gitignore b/claims_hosp/retrieve_files/.gitignore new file mode 100644 index 000000000..e69de29bb diff --git a/claims_hosp/setup.py b/claims_hosp/setup.py index 940e1d165..d7e46a13d 100644 --- a/claims_hosp/setup.py +++ b/claims_hosp/setup.py @@ -4,6 +4,7 @@ required = [ "numpy", "pandas", + "paramiko", "pydocstyle", "pytest", "pytest-cov", diff --git a/claims_hosp/tests/test_download_claims_ftp_files.py b/claims_hosp/tests/test_download_claims_ftp_files.py new file mode 100644 index 000000000..95f967ca3 --- /dev/null +++ b/claims_hosp/tests/test_download_claims_ftp_files.py @@ -0,0 +1,21 @@ +# standard +import datetime + +# third party +import numpy as np + +# first party +from delphi_claims_hosp.download_claims_ftp_files import (change_date_format, + get_timestamp) + + +class TestDownloadClaimsFtpFiles: + + def test_change_date_format(self): + name = "SYNEDI_AGG_INPATIENT_20200611_1451CDT" + expected = "SYNEDI_AGG_INPATIENT_11062020_1451CDT" + assert(change_date_format(name)==expected) + + def test_get_timestamp(self): + name = "SYNEDI_AGG_INPATIENT_20200611_1451CDT" + assert(get_timestamp(name).date() == datetime.date(2020, 6, 11)) diff --git a/claims_hosp/tests/test_get_latest_claims_name.py b/claims_hosp/tests/test_get_latest_claims_name.py new file mode 100644 index 000000000..ded5c9718 --- /dev/null +++ b/claims_hosp/tests/test_get_latest_claims_name.py @@ -0,0 +1,19 @@ +# standard +import time +from unittest.mock import Mock + +# third party +import pytest + + +from delphi_claims_hosp.get_latest_claims_name import get_latest_filename + + +class TestGetLatestFileName: + logger = Mock() + + def test_get_latest_claims_name(self): + dir_path = "./test_data/" + + with pytest.raises(AssertionError): + get_latest_filename(dir_path, self.logger) diff --git a/claims_hosp/tests/test_modify_claims_drops.py b/claims_hosp/tests/test_modify_claims_drops.py new file mode 100644 index 000000000..91d3d048f --- /dev/null +++ b/claims_hosp/tests/test_modify_claims_drops.py @@ -0,0 +1,19 @@ +# standard +from unittest.mock import Mock +from pathlib import Path + +# third party +from delphi_claims_hosp.modify_claims_drops import (modify_and_write) + + +class TestDropsModification: + + def test_modify_and_write(self): + data_path = "./test_data/" + logger = Mock() + files, dfs_list = modify_and_write(data_path, logger, test_mode=True) + expected_colnames = ['PatCountyFIPS', 'Pat HRR Name', 'Pat HRR ID', 'PatAgeGroup'] + assert len(files) == 1 + assert len(dfs_list) == 1 + assert files[0] == Path('./test_data/SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz') + assert set(expected_colnames).issubset(set(dfs_list[0].columns)) diff --git a/facebook/qsf-tools/combine_translations_eu.R b/facebook/qsf-tools/combine_translations_eu.R new file mode 100644 index 000000000..8ce09a353 --- /dev/null +++ b/facebook/qsf-tools/combine_translations_eu.R @@ -0,0 +1,95 @@ +#!/usr/bin/env Rscript + +## Combine a set of EU and non-EU translation files (UMD only), adding in a +## column indicating whether a given field was included in just the EU version, +## just the non-EU version, or in both. +## +## Usage: +## +## Rscript combine_translations_eu.R path/to/eu/translations/dir path/to/noneu/translations/dir path/to/combined/translations/dir + +suppressPackageStartupMessages({ + library(tidyverse) + source("qsf-utils.R") +}) + + +combine_translation_pair <- function(eu_translation, + noneu_translation) { + translation <- bind_rows(eu_translation, noneu_translation) %>% + mutate(eu_noneu = case_when( + startsWith(PhraseID, "intro1_eu") ~ "EU", + startsWith(PhraseID, "intro2_eu") ~ "EU", + startsWith(PhraseID, "intro1_noneu") ~ "nonEU", + startsWith(PhraseID, "intro2_noneu") ~ "nonEU", + TRUE ~ "Both" + )) + return(translation) +} + +combine_translations <- function(path_to_eu_translations, + path_to_noneu_translations, + path_to_combined) { + eu_name_pattern <- "_eu_" + if (!grepl(eu_name_pattern, path_to_eu_translations)) { + stop(path_to_eu_translations, "does not specify that it is for the EU") + } + noneu_name_pattern <- "_noneu_" + if (!grepl(noneu_name_pattern, path_to_noneu_translations)) { + stop(path_to_noneu_translations, "does not specify that it is for the non-EU") + } + + eu_files <- list.files(path_to_eu_translations, pattern = "*.csv$", full.names = TRUE) + eu_translations <- list() + for (filename in eu_files) { + eu_translations[[as.character(get_wave_from_csv(filename))]] <- read_csv(filename, show_col_types = FALSE) %>% + filter(startsWith(PhraseID, "intro")) + } + + noneu_files <- list.files(path_to_noneu_translations, pattern = "*.csv$", full.names = TRUE) + noneu_translations <- list() + for (filename in noneu_files) { + noneu_translations[[as.character(get_wave_from_csv(filename))]] <- read_csv(filename, show_col_types = FALSE) %>% + # Drop response options for the country + region question, they take up way too much space. + filter( + !startsWith(PhraseID, "A2_3_Answer"), + !startsWith(PhraseID, "A2_2_Answer"), + !startsWith(PhraseID, "NA_") + ) + } + + if (!identical(sort(names(eu_translations)), sort(names(noneu_translations)))) { + stop("not all waves are available for both EU and non-EU") + } + + dir.create(path_to_combined, showWarnings = FALSE) + for (wave in names(eu_translations)) { + combined <- combine_translation_pair( + eu_translations[[wave]], + noneu_translations[[wave]] + ) + + write_excel_csv( + combined, + file.path( + path_to_combined, + sprintf("umd_ctis_combined_v%02g_translations.csv", as.numeric(wave)) + ), + quote = "needed") + } +} + + +args <- commandArgs(TRUE) + +if (length(args) != 3) { + stop("Usage: Rscript combine_translations_eu.R path/to/eu/translations/dir path/to/noneu/translations/dir path/to/combined/translations/dir") +} + +path_to_eu_translations <- args[1] +path_to_noneu_translations <- args[2] +path_to_combined <- args[3] + +invisible(combine_translations(path_to_eu_translations, path_to_noneu_translations, path_to_combined)) + + diff --git a/facebook/qsf-tools/generate-codebook.R b/facebook/qsf-tools/generate-codebook.R index 26f81948e..17c5111e4 100644 --- a/facebook/qsf-tools/generate-codebook.R +++ b/facebook/qsf-tools/generate-codebook.R @@ -274,6 +274,7 @@ process_qsf <- function(path_to_qsf, # format all qsf content lists into a single tibble qdf <- tibble(variable = item_names, + qid = qids, question = questions, question_type = qtype, response_options = choices, @@ -360,6 +361,7 @@ process_qsf <- function(path_to_qsf, mutate(new = list( tibble(matrix_base_name = variable, variable = unlist(matrix_subquestion_field_names), + qid = qid, question = question, matrix_subquestion = unlist(matrix_subquestions), question_type = question_type, @@ -381,6 +383,7 @@ process_qsf <- function(path_to_qsf, mutate(new = list( tibble(matrix_base_name = variable, variable = unlist(matrix_subquestion_field_names), + qid = qid, question = question, matrix_subquestion = unlist(matrix_subquestions), question_type = question_type, @@ -426,6 +429,7 @@ process_qsf <- function(path_to_qsf, ) %>% select(wave, variable, + qid, matrix_base_name, replaces, description, diff --git a/facebook/qsf-tools/qsf-utils.R b/facebook/qsf-tools/qsf-utils.R index 2891964c1..4ec7280f3 100644 --- a/facebook/qsf-tools/qsf-utils.R +++ b/facebook/qsf-tools/qsf-utils.R @@ -71,6 +71,32 @@ get_wave <- function(path_to_qsf) { return(wave) } +#' Get wave number from qsf filename +#' +#' Wave number as provided in the qsf name should be an integer or a float with +#' one decimal place. +#' +#' @param path_to_file +#' +#' @return (mostly) integer wave number +get_wave_from_csv <- function(path_to_file) { + name_pattern <- "(.*[Ww]ave_?)([0-9]*([.][0-9])?)(.*csv.*)" + if (!grepl(name_pattern, path_to_file)) { + stop( + "The CSV filename must include the string 'csv', and the wave number in ", + "the format 'Wave_XX', 'WaveXX', 'wave_XX', or 'waveXX' where 'XX' is an ", + "integer or float. The wave specification can occur anywhere in the ", + "filename but must precede the string 'csv'." + ) + } + + wave <- as.numeric( + sub(name_pattern, "\\2", path_to_file) + ) + + return(wave) +} + #' Create mapping of QIDs to module name #' #' @param qsf contents of QSF file in JSON format diff --git a/facebook/qsf-tools/replace_translation_qids.R b/facebook/qsf-tools/replace_translation_qids.R new file mode 100644 index 000000000..ab769a4df --- /dev/null +++ b/facebook/qsf-tools/replace_translation_qids.R @@ -0,0 +1,73 @@ +#!/usr/bin/env Rscript + +## In translation CSVs, replace the QID in the name column with the human-readable +## item name (e.g. A1). Export modified translation CSVs in the same format as the +## original. +## +## Usage: +## +## Rscript replace_translation_qids.R path/to/translation/directory/or/single/translation/CSV path/to/codebook + +suppressPackageStartupMessages({ + library(tidyverse) + library(purrr) + library(stringr) + source("qsf-utils.R") +}) + + +replace_qid_wrapper <- function(path_to_translations, path_to_codebook) { + if (dir.exists(path_to_translations)) { + # Process all CSVs in directory + csvs <- list.files(path_to_translations, pattern = "*.csv$", full.names = TRUE) + for (csv in csvs) { + replace_qids(csv, path_to_codebook) + } + } else if (file.exists(path_to_translations)) { + replace_qids(path_to_translations, path_to_codebook) + } else { + stop(path_to_translations, " is not a valid file or directory") + } +} + +replace_qids <- function(path_to_translation_file, path_to_codebook) { + wave <- get_wave_from_csv(path_to_translation_file) + # Load codebook + codebook <- read_csv(path_to_codebook, col_types = cols( + .default = col_character(), + version = col_double() + )) %>% + filter(!is.na(qid), version == wave) + + # Load translation file + translation <- read_csv(path_to_translation_file, show_col_types = FALSE) %>% + # Drop survey ID line + filter(!startsWith(PhraseID, "SV_")) + + # Use codebook to make a mapping of QID -> item name. + var_qid_pairs <- codebook %>% mutate(variable = coalesce(matrix_base_name, variable)) %>% distinct(qid, variable) + qid_item_map <- var_qid_pairs %>% pull(variable) + names(qid_item_map) <- var_qid_pairs %>% pull(qid) + + # Use QID-name mapping to replace QID in first column. + ii_qid <- startsWith(translation$PhraseID, "QID") + translation[ii_qid,] <- translation[ii_qid,] %>% mutate( + PhraseID = str_replace(PhraseID, "(^QID[0-9]*)_", function(match) { + paste0(qid_item_map[str_sub(match, 1, -2)], "_") + }) + ) + + # Save processed file back to CSV under the same name. + write_excel_csv(translation, path_to_translation_file, quote = "needed") +} + +args <- commandArgs(TRUE) + +if (!(length(args) %in% c(2))) { + stop("Usage: Rscript replace_translation_qids.R path/to/translation/directory/or/single/translation/CSV path/to/codebook") +} + +path_to_translations <- args[1] +path_to_codebook <- args[2] + +invisible(replace_qid_wrapper(path_to_translations, path_to_codebook)) diff --git a/facebook/qsf-tools/static/UMD/item_shortquestion_map.csv b/facebook/qsf-tools/static/UMD/item_shortquestion_map.csv index 8286ae9d5..f7a207255 100644 --- a/facebook/qsf-tools/static/UMD/item_shortquestion_map.csv +++ b/facebook/qsf-tools/static/UMD/item_shortquestion_map.csv @@ -1,178 +1,156 @@ item,description -intro1_eu,participation EU -intro2_eu,data share EU -intro1_noneu,participation nonEU -intro2_noneu,data share nonEU -intro1_noneu_jhu,TODO -intro1_eu_jhu, TODO -intro1_eu_jhsph, TODO -intro1_row_jhu, TODO -intro1_row_jhsph, TODO +1w_0unw,if country is weighted A1,18yrs+ -A1_2,agecheckV2 TODO +A1_2,consent age A2,country/region +A2_1120,country + region +A2_1220,country + region A2_2,country/region -A2_3,administrative region -A3, TODO A2_2_1,country/region A2_2_2,administrative region +A2_3,administrative region +A3, postal code +B0,ever had covid B0a,ever had covid B1,symptoms +B10,reduce spendings pay for test +B11,wanted test 14d +B11a,wanted test 24h +B11b,wanted to test sympdays +B11c,wanted test 14d +B12,reason not tested 14d +B12a_likert,reason not tested 14d +B12a_profile,reason not tested 14d +B12b,reason not tested 14d +B12b_likert,reason not tested 14d +B12b_profile,reason not tested 14d +B12c,reason not tested 14d +B12c_matrix,reason not tested 14d matrix +B13,needed med services 30d +B13_likert,needed med services 30d +B13_profile,needed med services 30d +B14,reason no med services 30d +B14_likert,reason no med services 30d +B14_profile,reason no med services 30d +B15,reason tested 14d +B1_matrix,symptoms B1b,unusual symptoms B1b_likert,unusual symptoms -B1_matrix, TODO -B1b_matrix, TODO +B1b_matrix,unusual symptoms B2,unusual symptoms num days -B2a,unusual symptoms num days TODO -B2b,unusual symptoms num days TODO -B2c,unusual symptoms num days TODO -B3,community sick +B2a,unusual symptoms num days +B2b,unusual symptoms num days +B2c,unusual symptoms num days +B3,community sick B4,community num sick B5,spent time with sick B6,tested ever B7,tested 14d -B7a,tested TODO -B7b,tested TODO -B7c,tested TODO +B7a,tested past day +B7b,tested sympdays +B7c,tested 14d B8,positive test 14d B8a,most recent test positive B9,pay for test -B9_alt1,TODO -B9_alt2,TODO -B9_alt3,TODO -B10,reduce spendings pay for test -B11,wanted test 14d -B11a,TODO -B11b,TODO -B11c,TODO -B12,reason not tested 14d -B12b,reason not tested 14d -B12c,TODO -B12c_matrix,TODO -B12a_likert,TODO -B12a_profile,TODO -B12b_likert,TODO -B12b_profile,TODO -B13,needed med services 30d -B13_likert,TODO -B13_profile,TODO -B14,reason no med services 30d -B14_likert,TODO -B14_profile,TODO -B15,reason tested 14d +B9_alt1,pay for test +B9_alt2,pay for test +B9_alt3,pay for test C0,activities 24h -C0a,activities 24h C0_likert,activities 24h -C0_matrix,TODO +C0_matrix,activities 24h +C0a,activities 24h +C10,flu vaccine intend by Jan 2020 +C11_no,reasons no flu vaccine +C11_no_8_TEXT,reasons no flu vaccine other text +C11_unsure,reasons unsure flu vaccine +C11_unsure_7_TEXT,reasons unsure flu vaccine other text +C12,flu vaccine june2019 to feb2020 +C13,mask worn 24h +C13a,mask worn 24h indoors +C14,avoid contact +C14a,avoid contact 7d +C1_10,direct contact 10min outsider 24h +C1_15,direct contact 15min outsider 24h +C1_5,direct contact 5min outsider 24h C1_m,direct contact outsider 24h -C1_5,TODO -C1_10,TODO -C1_15,TODO C2,num contact outsider 24h -C2_2a, TODO -C2_2b, TODO -C2_2c, TODO -C2_oe, TODO +C2_2a,direct contact outsider 24h +C2_2b,direct contact outsider 24h open ended +C2_2c,direct contact outsider 24h +C2_oe,direct contact outsider 24h open ended C3,spent time health clinic/hospital 7d C4,washing hands after public 7d -C4_2,TODO +C4_2,washing hands after public 7d C5,wear mask in public 7d -C5_2,TODO -C6,days spent with outsider 7d -C6_2,TODO +C5_2,wear mask in public 7d +C6,days spent with outsider 7d +C6_2,days spent with outsider 7d C7,num washed hands 24d -C7_oe,TODO -C8,access soap water +C7_oe,num washed hands 24d open ended +C8,access soap water C9,flu vaccine since June 2020 C9a,flu vaccine since July 2020 -C10,flu vaccine intend by Jan 2020 -C11_no,reasons no flu vaccine -C11_unsure,reasons unsure flu vaccine -C12,flu vaccine june2019 to feb2020 -C13a,mask worn 24h indoors -C13,mask worn 24h -C14,avoid contact -C14a,avoid contact 7d +country_agg,country name +country_region_numeric,country region option D1,nervousness 7d -D1_14,TODO +D10a,main activity occupation +D10b,main activity occupation before Feb 2020 +D1_14,nervousness 14d D2,depressed 7d -D2_14,TODO -D2_30_cheer,TODO -D2_30_calm,TODO +D2_14,depressed 14d D3,worried ill D4,worried food next week D5,worried finances next month D6,reasons worried finances -D6_likert,TODO +D6_likert,reasons worried finances likert D7,paid work 7d D7a,paid work 4w D8,paid work before feb 2020 D9,reason stopped working -D9_alt,TODO -D10,main activity occupation -D10a,main activity occupation -D10b,TODO +D9_alt,reason stopped working E2,area type E3,gender E4,age E5,num ppl slept where stayed E6,num yrs education E7,rooms for sleep multichoice -E7_oe,TODO +E7_oe,rooms for sleep open ended E7a,num rooms for sleep fill in E8,education level F1,smartphone F2,tracking app -F2_likert,TODO +F2_likert,tracking app likert F3,tracking app F3_au,COVIDSafe app F3_de,Corona-Warn-App -F3_uk,NHS app TODO -V1,vaccinated -V1alt_A,know anyone vaccinated -V2,vaccine doses -V2a,inital doses -V2b,booster doses -V2c,booster intend -V3,vaccine accepting likert -V3a,vaccine accepting likert 2 -V4,trust vaccine recommend source -V4_fixed,trust vaccine recommend source -V9,concern side effect -V5a,vaccine hesitancy reasons prob yes -V5b,vaccine hesitancy reasons prob no -V5c,vaccine hesitancy reasons def no -V5d,vaccine hesitancy reasons not all doses -V6,reasons vaccination not needed -V10,chronic conditions -V11,pregnant -V12,smoke -V13,informed vaccine access -V15,have vax appointment -V15a,have vax appointment 2 -V16,tried get vaccine appointment -V16a,tried get vaccine appointment 2 -V18a,vaccination barriers vaccinated ever 1 -V18b,vaccination barriers unvaccinated -V18c,vaccination barriers vaccinated 2 -V19,when try to get vaccine -V20,"recent vaccination date intro" -V20_month,recent vaccination month -V20_year,recent vaccination year +F3_uk,NHS app +Finished,Facebook respondent weight G1,worry catch covid G2,belief social distancing G3,belief wearing mask +GID_0,country code by gdam +GID_1,region code by gdam H1,others maintained distance 7d H2,others wear masks 7d H3,friends family vaccination I1,belief vaccinated stop masking +I10_noneu,info channels trust I2,belief children no covid I5,covid news sources 7d I6,covid news sources trust I7,additional covid topics I8,belief no covid hot climate I9_noneu,info channels 7d -I10_noneu,info channels trust +intro1_eu,participation EU +intro1_eu_jhsph,consent eu jhsph +intro1_eu_jhu,consent eu jhu +intro1_noneu,participation nonEU +intro1_noneu_jhu,consent noneu jhu +intro1_row_jhsph,consent noneu jhsph +intro1_row_jhu,consent noneu jhu +intro2_eu,data share EU +intro2_noneu,data share nonEU +ISO_3,country code in ISO_3 standard J1,children <18 household J2,vaccinate children J3,parent or guardian @@ -182,7 +160,48 @@ J6,oldest child schooling J7,remote schooling J8,oldest child gender K1,delayed medical care cost 1y -Vaccine Text,vaccine dose explanation +module,question set respondent was assigned to +NAME_0,country name by gdam +NAME_1,region name by gdam Q75,reference oldest child -A2_1120,country + region -A2_1220,country + region +Q_Language,survey language +Q_TotalDuration,duration of response in seconds +RecordedDate,survey timestamp that the response was recorded in Pacific time (UTC-7) +region_agg,region name +survey_region,survey version +survey_version,survey version +V1,vaccinated +V10,chronic conditions +V11,pregnant +V12,smoke +V13,informed vaccine access +V15,have vax appointment +V15a,have vax appointment 2 +V16,tried get vaccine appointment +V16a,tried get vaccine appointment 2 +V18a,vaccination barriers vaccinated ever 1 +V18b,vaccination barriers unvaccinated +V18c,vaccination barriers vaccinated 2 +V19,when try to get vaccine +V1alt_A,know anyone vaccinated +V2,vaccine doses +V20,recent vaccination date intro +V20_month,recent vaccination month +V20_year,recent vaccination year +V2a,inital doses +V2b,booster doses +V2c,booster intend +V3,vaccine accepting likert +V3a,vaccine accepting likert 2 +V4,trust vaccine recommend source +V4_fixed,trust vaccine recommend source +V5a,vaccine hesitancy reasons prob yes +V5b,vaccine hesitancy reasons prob no +V5c,vaccine hesitancy reasons def no +V5d,vaccine hesitancy reasons not all doses +V6,reasons vaccination not needed +V9,concern side effect +Vaccine Text,vaccine dose explanation +w12_treatment,experimental treatment respondent was assigned to +weight,Facebook respondent weight +weight_type,Facebook respondent weight \ No newline at end of file diff --git a/sir_complainsalot/params.json.template b/sir_complainsalot/params.json.template index 9eeb06123..735e7f7bd 100644 --- a/sir_complainsalot/params.json.template +++ b/sir_complainsalot/params.json.template @@ -43,100 +43,6 @@ "maintainers": ["U01AP8GSWG3","U01069KCRS7"], "retired-signals": ["completely_home_prop", "full_time_work_prop", "part_time_work_prop", "median_home_dwell_time", "completely_home_prop_7dav", "full_time_work_prop_7dav", "part_time_work_prop_7dav", "median_home_dwell_time_7dav"] }, - "fb-survey": { - "max_age": 3, - "maintainers": ["U01069KCRS7"], - "retired-signals": [ - "smoothed_anxious_5d", "smoothed_wanxious_5d", - "smoothed_depressed_5d", "smoothed_wdepressed_5d", - "smoothed_felt_isolated_5d", "smoothed_wfelt_isolated_5d", - "smoothed_large_event_1d", "smoothed_wlarge_event_1d", - "smoothed_restaurant_1d", "smoothed_wrestaurant_1d", - "smoothed_shop_1d", "smoothed_wshop_1d", - "smoothed_spent_time_1d", "smoothed_wspent_time_1d", - "smoothed_travel_outside_state_5d", "smoothed_wtravel_outside_state_5d", - "smoothed_work_outside_home_1d", "smoothed_wwork_outside_home_1d", - "smoothed_wearing_mask", "smoothed_wwearing_mask", - "smoothed_vaccine_likely_local_health", "smoothed_wvaccine_likely_local_health", - "smoothed_others_masked", "smoothed_wothers_masked", - "smoothed_wanted_test_14d", "smoothed_wwanted_test_14d", - "smoothed_covid_vaccinated_or_accept", "smoothed_wcovid_vaccinated_or_accept", - "smoothed_accept_covid_vaccine", "smoothed_waccept_covid_vaccine", - "smoothed_hesitancy_reason_allergic", "smoothed_whesitancy_reason_allergic", - "smoothed_hesitancy_reason_not_recommended", "smoothed_whesitancy_reason_not_recommended", - "smoothed_hesitancy_reason_distrust_vaccines", "smoothed_whesitancy_reason_distrust_vaccines", - "smoothed_hesitancy_reason_health_condition", "smoothed_whesitancy_reason_health_condition", - "smoothed_hesitancy_reason_pregnant", "smoothed_whesitancy_reason_pregnant", - "smoothed_vaccine_likely_friends", "smoothed_wvaccine_likely_friends", - "smoothed_vaccine_likely_who", "smoothed_wvaccine_likely_who", - "smoothed_vaccine_likely_govt_health", "smoothed_wvaccine_likely_govt_health", - "smoothed_vaccine_likely_politicians", "smoothed_wvaccine_likely_politicians", - "smoothed_vaccine_likely_doctors", "smoothed_wvaccine_likely_doctors", - "smoothed_felt_isolated_7d", "smoothed_wfelt_isolated_7d", - "smoothed_worried_become_ill", "smoothed_wworried_become_ill", - "smoothed_received_2_vaccine_doses", "smoothed_wreceived_2_vaccine_doses", - "smoothed_hesitancy_reason_dislike_vaccines", "smoothed_whesitancy_reason_dislike_vaccines", - "smoothed_inperson_school_fulltime", "smoothed_winperson_school_fulltime", - "smoothed_inperson_school_parttime", "smoothed_winperson_school_parttime", - "smoothed_vaccinate_children", "smoothed_wvaccinate_children", - ["smoothed_vaccine_barrier_appointment_time_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_appointment_time_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_childcare_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_childcare_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_document_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_document_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_eligible_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_eligible_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_language_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_language_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_no_appointments_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_no_appointments_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_none_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_none_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_technical_difficulties_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_technical_difficulties_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_technology_access_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_technology_access_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_time_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_time_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_travel_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_travel_tried", "msa", "county", "state"], - ["smoothed_vaccine_barrier_type_tried", "msa", "county", "state"], ["smoothed_wvaccine_barrier_type_tried", "msa", "county", "state"], - ["smoothed_try_vaccinate_1m", "hrr"], ["smoothed_wtry_vaccinate_1m", "hrr"], - ["smoothed_try_vaccinate_1m", "msa"], ["smoothed_wtry_vaccinate_1m", "msa"], - ["smoothed_dontneed_reason_dont_spend_time", "hrr"], ["smoothed_wdontneed_reason_dont_spend_time", "hrr", "msa"], - ["smoothed_dontneed_reason_had_covid", "hrr"], ["smoothed_wdontneed_reason_had_covid", "hrr", "msa"], - ["smoothed_dontneed_reason_not_beneficial", "hrr"], ["smoothed_wdontneed_reason_not_beneficial", "hrr", "msa"], - ["smoothed_dontneed_reason_not_high_risk", "hrr"], ["smoothed_wdontneed_reason_not_high_risk", "hrr", "msa"], - ["smoothed_dontneed_reason_not_serious", "hrr"], ["smoothed_wdontneed_reason_not_serious", "hrr", "msa"], - ["smoothed_dontneed_reason_other", "hrr"], ["smoothed_wdontneed_reason_other", "hrr", "msa"], - ["smoothed_dontneed_reason_precautions", "hrr"], ["smoothed_wdontneed_reason_precautions", "hrr", "msa"], - "smoothed_screening_tested_positive_14d", "smoothed_wscreening_tested_positive_14d", - "smoothed_travel_outside_state_7d", "smoothed_wtravel_outside_state_7d", - "smoothed_belief_vaccinated_mask_unnecessary", "smoothed_wbelief_vaccinated_mask_unnecessary", - "smoothed_belief_children_immune", "smoothed_wbelief_children_immune", - "smoothed_received_2_vaccine_doses", "smoothed_wreceived_2_vaccine_doses", - "smoothed_vaccine_barrier_eligible", "smoothed_wvaccine_barrier_eligible", - "smoothed_vaccine_barrier_no_appointments", "smoothed_wvaccine_barrier_no_appointments", - "smoothed_vaccine_barrier_appointment_time", "smoothed_wvaccine_barrier_appointment_time", - "smoothed_vaccine_barrier_technical_difficulties", "smoothed_wvaccine_barrier_technical_difficulties", - "smoothed_vaccine_barrier_document", "smoothed_wvaccine_barrier_document", - "smoothed_vaccine_barrier_technology_access", "smoothed_wvaccine_barrier_technology_access", - "smoothed_vaccine_barrier_travel", "smoothed_wvaccine_barrier_travel", - "smoothed_vaccine_barrier_language", "smoothed_wvaccine_barrier_language", - "smoothed_vaccine_barrier_childcare", "smoothed_wvaccine_barrier_childcare", - "smoothed_vaccine_barrier_time", "smoothed_wvaccine_barrier_time", - "smoothed_vaccine_barrier_type", "smoothed_wvaccine_barrier_type", - "smoothed_vaccine_barrier_none", "smoothed_wvaccine_barrier_none", - "smoothed_vaccine_barrier_appointment_location", "smoothed_wvaccine_barrier_appointment_location", - "smoothed_vaccine_barrier_other", "smoothed_wvaccine_barrier_other", - "smoothed_vaccine_barrier_eligible_has", "smoothed_wvaccine_barrier_eligible_has", - "smoothed_vaccine_barrier_no_appointments_has", "smoothed_wvaccine_barrier_no_appointments_has", - "smoothed_vaccine_barrier_appointment_time_has", "smoothed_wvaccine_barrier_appointment_time_has", - "smoothed_vaccine_barrier_technical_difficulties_has", "smoothed_wvaccine_barrier_technical_difficulties_has", - "smoothed_vaccine_barrier_document_has", "smoothed_wvaccine_barrier_document_has", - "smoothed_vaccine_barrier_technology_access_has", "smoothed_wvaccine_barrier_technology_access_has", - "smoothed_vaccine_barrier_travel_has", "smoothed_wvaccine_barrier_travel_has", - "smoothed_vaccine_barrier_language_has", "smoothed_wvaccine_barrier_language_has", - "smoothed_vaccine_barrier_childcare_has", "smoothed_wvaccine_barrier_childcare_has", - "smoothed_vaccine_barrier_time_has", "smoothed_wvaccine_barrier_time_has", - "smoothed_vaccine_barrier_type_has", "smoothed_wvaccine_barrier_type_has", - "smoothed_vaccine_barrier_none_has", "smoothed_wvaccine_barrier_none_has", - "smoothed_vaccine_barrier_appointment_location_has", "smoothed_wvaccine_barrier_appointment_location_has", - "smoothed_vaccine_barrier_other_has", "smoothed_wvaccine_barrier_other_has", - ["smoothed_vaccine_barrier_appointment_location_tried", "county", "state"], ["smoothed_wvaccine_barrier_appointment_location_tried", "county", "state"], - ["smoothed_vaccine_barrier_other_tried", "county", "state"], ["smoothed_wvaccine_barrier_other_tried", "county", "state"] - ] - }, "quidel": { "max_age":6, "maintainers": ["U01AP8GSWG3","U01069KCRS7"],